def run_demo(net, image_provider, height_size, cpu):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
color = [0, 224, 255]
# buf = []
finish = False
begun = False
count = 0
avgwes, avgesh, avgshk, avgseh, avghorizontal, avgsek = 0., 0., 0., 0., 0., 0.
badpose_startt = 0
pose_startt = 0
total_bad_duration = 0
lower_hip_time = 0
averaging_duration = 3
message_time_gap = 7
reset_time_gap = 5
all_checks = (Checker(message_time_gap, reset_time_gap, bodyDownCheck,
"Raise your body up"),
Checker(message_time_gap, reset_time_gap, kneeDownCheck,
"Raise your knees"),
Checker(message_time_gap, reset_time_gap, backHighCheck,
"Lower your back"))
avg_accuracy = AverageAccuracy()
#Stop the workout if the an incorrect posture lasts for more than 15 seconds
max_bad_pose_duration = 20
last_time_time = time.time()
knee_touching_ground = False
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(18): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
# print(len(pose_entries))
# print("parts__")
kpLocations = dict()
# print("Pose entries: ", len(pose_entries))
for n in range(len(pose_entries)):
# print("pose entries length ", len(pose_entries[n]))
if len(pose_entries[n]) < 10:
continue
if not checkDistance(pose_entries[n], all_keypoints, "left",
"Shoulder", "Ankle"):
continue
if not checkDistance(pose_entries[n], all_keypoints, "left",
"Elbow", "Hip"):
continue
if not checkDistance(pose_entries[n], all_keypoints, "left",
"Shoulder", "Hip"):
continue
if not checkDistance(pose_entries[n], all_keypoints, "left",
"Shoulder", "Knee"):
continue
if not checkDistance(pose_entries[n], all_keypoints, "left",
"Shoulder", "Ankle"):
continue
if not checkDistance(pose_entries[n], all_keypoints, "left",
"Elbow", "Hip"):
continue
if not checkDistance(pose_entries[n], all_keypoints, "left",
"Elbow", "Knee"):
continue
# print("Is horizontal : " , horizontal)
left = isFacingLeft(pose_entries[n], all_keypoints)
print("Is facing left ", left)
anglehorizontal = horizontalAngle(pose_entries[n], all_keypoints)
displayText("Hor: " + str(anglehorizontal), 50, 20, img)
anglewes, angleesh, angleshk, angleseh, anglesek = getPlankAngles(
pose_entries[n], all_keypoints, img)
cont = False
for angl in (anglewes, angleesh, angleshk, angleseh, anglesek):
if math.isnan(angl):
cont = True
print("nan found")
break
if cont:
continue
avghorizontal += anglehorizontal
avgwes += anglewes
avgesh += angleesh
avgshk += angleshk
avgseh += angleseh
avgsek += anglesek
count += 1
if count == averaging_duration:
count = 0
avgwes /= averaging_duration
avgesh /= averaging_duration
avgshk /= averaging_duration
avgseh /= averaging_duration
avghorizontal /= averaging_duration
avgsek /= averaging_duration
horizontal = checkHorizontal(avghorizontal)
# print("Is horizontal ", horizontal)
if horizontal:
# print("Avg Angles: wrist elbow shoulder: {0} elbow shoulder hips: {1} shoulder hips knee: {2} shoulder elbow hip {3}".format(avgwes, avgesh, avgshk, avgseh))
correct_pose, wes, esh, shk, seh = checkCorrectPlank(
avgwes, avgesh, avgshk, avgseh)
if not begun and correct_pose:
begun = True
pose_startt = time.time()
last_time_time = time.time()
tts.BotSpeak(
"Now that you are in position, I'll start the timer."
)
botlog_queue.put(
"Now that you are in position, I'll start the timer."
)
if begun:
accuracy = getAccuracy(avgwes, avgesh, avgshk, avgseh,
avgsek)
accuracy_queue.put(accuracy)
avg_accuracy.put(accuracy)
displayText(accuracy, 10, 120, img)
correct_pose = True
for check in all_checks:
if not check.check(avgwes, avgesh, avgshk, avgseh,
avgsek):
# if check.display_message:
# tts.BotSpeak(check.message)
if check.display_message:
tts.BotSpeak(check.message)
botlog_queue.put(check.message)
displayText(check.message, 20, 100, img,
(0, 255, 0))
correct_pose = False
break
d = time.time() - last_time_time
if d > 10:
seconds_over = int(time.time() - pose_startt)
tts.BotSpeak(
"{0} seconds over".format(seconds_over))
botlog_queue.put(
"{0} seconds over".format(seconds_over))
last_time_time = time.time()
if not correct_pose:
# if not shk:
# if lower_hip_time > 0:
# d = time.time() - lower_hip_time
# if d > 5:
# tts.BotSpeak("Lower your hips")
# lower_hip_time = 0
# else:
# lower_hip_time = time.time()
# else:
# lower_hip_time = 0
if badpose_startt != 0:
duration = time.time() - badpose_startt
displayText(
"badpose duration: " + str(duration), 0,
50, img)
if duration > max_bad_pose_duration:
finish = True
# if duration > 20:
# displayText("Pose broken for 20", 0, 40, img)
else:
badpose_startt = time.time()
else:
lower_hip_time = 0
if badpose_startt != 0:
total_bad_duration += time.time(
) - badpose_startt
badpose_startt = 0
displayText("Correct Pose", 0, 30, img)
print("Correct Pose: ", correct_pose)
elif begun:
finish = True
avgwes, avgesh, avgshk, avgseh, avghorizontal, avgsek = 0., 0., 0., 0., 0., 0.
if finish:
if badpose_startt > 0:
total_bad_duration += time.time() - badpose_startt
duration = time.time() - pose_startt
good_duration = duration - total_bad_duration
tts.BotSpeak("You held an incorrect position for too long.")
botlog_queue.put(
"You held an incorrect position for too long.")
tts.BotSpeak("Your workout is now complete")
botlog_queue.put("Your workout is now complete")
tts.BotSpeak("Your average accuracy was {0}".format(
int(avg_accuracy.get())))
botlog_queue.put("Your average accuracy was {0}".format(
int(avg_accuracy.get())))
tts.BotSpeak(
"You held the correct plank position for {0} seconds and your workout lasted for {1} seconds."
.format(int(good_duration), int(duration)))
botlog_queue.put(
"You held the correct plank position for {0} seconds and your workout lasted for {1} seconds."
.format(int(good_duration), int(duration)))
# tts.BotSpeak("end")
print("-" * 20)
print("Total duration: ", good_duration)
return
# print("true_count {0} count {1} finish {2} begun {3} ".format(true_count, count, finish, begun))
# triplet = ("left", "Shoulder", "Hip", "Ankle")
# # triplet = ("left", "Wrist", "Elbow", "Shoulder")
# print(getAngle(pose_entries[n], all_keypoints, *triplet))
# print("wes", getAngle(pose_entries[n], all_keypoints, "left", "Wrist", "Elbow", "Shoulder"))
# print("esh", getAngle(pose_entries[n], all_keypoints, "left", "Elbow", "Shoulder", "Hip"))
# drawLinesTriplets(pose_entries[n], all_keypoints, img, color, *triplet)
# print("trip2", getAngle(pose_entries[n], all_keypoints, "", "leftShoulder", "chestCenter", "rightShoulder")/180)
for part_id in range(len(BODY_PARTS_PAF_IDS) - 2):
kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0]
global_kpt_a_id = pose_entries[n][kpt_a_id]
if global_kpt_a_id != -1:
x_a, y_a = all_keypoints[int(global_kpt_a_id), 0:2]
cv2.circle(img, (int(x_a), int(y_a)), 3, color, -1)
# cv2.putText(img, str(kpt_a_id),
# (int(x_a), int(y_a)),
# font,
# fontScale,
# fontColor,
# lineType)
kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1]
global_kpt_b_id = pose_entries[n][kpt_b_id]
if global_kpt_b_id != -1:
x_b, y_b = all_keypoints[int(global_kpt_b_id), 0:2]
cv2.circle(img, (int(x_b), int(y_b)), 3, color, -1)
if global_kpt_a_id != -1 and global_kpt_b_id != -1:
cv2.line(img, (int(x_a), int(y_a)), (int(x_b), int(y_b)),
color, 2)
# print(part_id, (x_a, y_a), (x_b, y_b))
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
# cv2.imwrite("output.jpeg", img)
key = cv2.waitKey(33)
if key == 27: # esc
# tts.BotSpeak("end")
return
def run_demo(net, action_net, image_provider, height_size, cpu, boxList):
net = net.eval()
print(torch.cuda.device_count())
print(torch.cuda.is_available())
a = torch.Tensor(5, 3)
a = a.cuda()
print(a)
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts # 18
i = 0
for img in image_provider: # 遍历图像集
orig_img = img.copy() # copy 一份
# print(i)
fallFlag = 0
if i % 1 == 0:
heatmaps, pafs, scale, pad = infer_fast(
net, img, height_size, stride, upsample_ratio,
cpu) # 返回热图,paf,输入模型图象相比原始图像缩放倍数,输入模型图像padding尺寸
total_keypoints_num = 0
all_keypoints_by_type = [
] # all_keypoints_by_type为18个list，每个list包含Ni个当前点的x、y坐标，当前点热图值，当前点在所有特征点中的index
for kpt_idx in range(
num_keypoints): # 19th for bg 第19个为背景，之考虑前18个关节点
total_keypoints_num += extract_keypoints(
heatmaps[:, :, kpt_idx], all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(
all_keypoints_by_type, pafs, demo=True
) # 得到所有分配的人（前18维为每个人各个关节点在所有关节点中的索引，后两唯为每个人得分及每个人关节点数量），及所有关节点信息
for kpt_id in range(all_keypoints.shape[0]): # 依次将每个关节点信息缩放回原始图像上
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)): # 依次遍历找到的每个人
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones(
(num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
posebox = (int(pose.bbox[0]), int(pose.bbox[1]),
int(pose.bbox[0]) + int(pose.bbox[2]),
int(pose.bbox[1]) + int(pose.bbox[3]))
coincideValue = coincide(boxList, posebox)
print(posebox)
print('coincideValue:' + str(coincideValue))
if len(
pose.getKeyPoints()
) >= 10 and coincideValue >= 0.3 and pose.lowerHalfFlag < 3: # 当人体的点数大于10个的时候算作一个人,同时判断yolov5的框和pose的框是否有交集并且占比30%,同时要有下半身
current_poses.append(pose)
for pose in current_poses:
pose.img_pose = pose.draw(img, is_save=True, show_draw=True)
crown_proportion = pose.bbox[2] / pose.bbox[3] #宽高比
pose = action_detect(action_net, pose,
crown_proportion) #判断摔倒还是正常
if pose.pose_action == 'fall':
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2],
pose.bbox[1] + pose.bbox[3]), (0, 0, 255),
thickness=3)
cv2.putText(img, 'state: {}'.format(pose.pose_action),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
fallFlag = 1
else:
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2],
pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
cv2.putText(img, 'state: {}'.format(pose.pose_action),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0))
# fallFlag = 1
if fallFlag == 1:
t = time.time()
# cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img)
print('我保存照片了')
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
# 保存识别后的照片
# cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img)
# print('我保存照片了')
# cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
cv2.waitKey(1)
i += 1
cv2.destroyAllWindows()
def run_demo(net, image_provider, height_size, cpu, track_ids, arm):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
stateMachines = {}
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])
"""
kpt_names = ['nose', 'neck',
'r_sho', 'r_elb', 'r_wri', 'l_sho', 'l_elb', 'l_wri',
'r_hip', 'r_knee', 'r_ank', 'l_hip', 'l_knee', 'l_ank',
'r_eye', 'l_eye',
'r_ear', 'l_ear']
r_elb-3, r-wri-4, l_elb-6, l_wri-7
"""
# print('ID: {}'.format(n))
# print('\tRight elbow: {}, right wrist: {}'.format(pose_keypoints[3], pose_keypoints[4]))
# print('\tLeft elbow: {}, left wrist: {}'.format(pose_keypoints[6], pose_keypoints[7]))
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
if track_ids == True:
propagate_ids(previous_poses, current_poses, threshold=3)
previous_poses = current_poses
for pose in current_poses:
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
for pose in current_poses:
if pose.id not in stateMachines.keys():
stateMachines[pose.id] = StateMachine(pose.id, pose.keypoints, arm)
#print('ID {} detected'.format(pose.id))
continue
# call stateMachine methods
stateMachines[pose.id].update(pose.keypoints, img)
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(33)
if key == 27: # esc
return
def run_demo(net, image_provider, height_size, cpu, track, smooth):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 33
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
# print("showing")
cv2.imwrite(f"outputs/demo-{image_provider.idx}.jpg", img)
def run_demo(model, image_provider, height_size, cpu, track, smooth, file):
"""[summary]
Args:
model ([type]): [description]
image_provider ([type]): [description]
height_size ([type]): [description]
cpu ([type]): [description]
track ([type]): [description]
smooth ([type]): [description]
file ([type]): [description]
Returns:
[type]: [description]
"""
model = model.eval()
if not cpu:
model = model.cuda()
point_list = []
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
# 保存视频
fps = image_provider.fps
width = image_provider.width
height = image_provider.height
fourcc = cv2.VideoWriter_fourcc('M', 'P', '4', 'V')
# video_saver = cv2.VideoWriter('TESTV.mp4', fourcc, fps, (height, width))
save_video_path = os.path.join(os.getcwd(), 'video_output')
if not os.path.exists(save_video_path):
os.mkdir(save_video_path)
save_video_name = os.path.join(save_video_path, file + '.mp4')
video_saver = cv2.VideoWriter(save_video_name, fourcc, fps,
(width, height))
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(model, img, height_size,
stride, upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for pose_entry in pose_entries:
if len(pose_entry) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entry[kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entry[kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entry[kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entry[18])
current_poses.append(pose)
# save keypoint in list
key_point_list = pose_keypoints.flatten().tolist()
point_list.append(key_point_list)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
video_saver.write(img)
return point_list
def run_demo(net, image_provider, height_size, cpu, track_ids):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu)
# Debug code
for j in range(0, 16):
heatmap = heatmaps[:,:,j]
heatmap = heatmap.reshape((128,128,1))
heatmapimg = np.array(heatmap * 255, dtype = np.uint8)
heatmap = cv2.applyColorMap(heatmapimg, cv2.COLORMAP_JET)
#heatmap = heatmap/255
cv2.imwrite('hmtestpadh_'+str(j)+'.jpg', heatmap)
#h_img = TF.to_pil_image(heatmaps)
#h_img.save('img_heatmap', 'JPEG')
print('heatmaps shape: {}'.format(heatmaps.shape))
np.savetxt('heatmaps.txt', heatmaps.reshape(-1))
rows, cols, depths = np.nonzero(pafs)
print(pafs[rows, cols, depths])
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num)
# Debug code
print('total_keypoints_num: {} {} {}\n'.format(all_keypoints_by_type, total_keypoints_num, kpt_idx))
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
if track_ids == True:
propagate_ids(previous_poses, current_poses)
previous_poses = current_poses
for pose in current_poses:
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(0)
if key == 27: # esc
return
def run_demo(net, image_provider, height_size, cpu, track_ids): #, filename):
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts #18개
previous_poses = []
c = 0
ttt = 0
for img in image_provider:
t5 = time.time()
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]): ##사이즈 변환
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
if track_ids == True: ##Track Poses
propagate_ids(previous_poses, current_poses)
previous_poses = current_poses
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
tt = time.time()
fps = 1 / (tt - ttt)
print('fps=', fps)
ttt = time.time()
str = "FPS : %0.2f" % fps
cv2.putText(img, str, (0, 100), cv2.FONT_HERSHEY_SIMPLEX, 1,
(0, 255, 0))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
# cv2.imwrite('PAF_'+filename[image_provider.idx -1], img)
key = cv2.waitKey(1)
if key == 27: # esc
return
def run_demo(net, image_provider, height_size, cpu, track, smooth):
show_time = True
net = net.eval()
if not cpu:
net = net.cuda()
track = True
print("track:" + str(track))
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 1
for img, img_dir in image_provider:
orig_img = img.copy()
start_time = datetime.datetime.now()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
end_time = datetime.datetime.now()
if show_time:
print("pose time:")
print((end_time - start_time).microseconds / 1000)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
#cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
print(img_dir)
img_save_dir = img_dir.replace("15_1", "15_1_lightpose")
#cv2.imwrite("/data2/qilei_chen/DATA/ShanghaiAutograding/gangganpingheng_images_240/15_1/right_light_pose/"+os.path.basename(img_dir),img)
cv2.imwrite(img_save_dir, img)
'''
def run_demo(net, image_provider, height_size, cpu, track, smooth):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 1
mean_time = 0
video_time = 0
prep_displacement = 0
prep_time = 0
for img in image_provider:
tik1 = cv2.getTickCount()
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
if (pose.bbox[0] + pose.bbox[2] / 2 > 355
and (pose.bbox[0] + pose.bbox[2] / 2 < 555)):
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
cv2.rectangle(img, (330, 330), (430, 230), (0, 0, 255), 2)
prev_pose = []
for pose in current_poses:
if track:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
if (pose.keypoints[4][0] > 330 and pose.keypoints[4][0] < 430) and (pose.keypoints[4][1] > 230 and pose.keypoints[4][1] < 330) and \
(pose.keypoints[7][0] > 330 and pose.keypoints[7][0] < 430) and (pose.keypoints[7][1] > 230 and pose.keypoints[7][1] < 330):
cv2.rectangle(
img,
(pose.keypoints[4][0] - 10, pose.keypoints[4][1] - 10),
(pose.keypoints[4][0] + 10, pose.keypoints[4][1] + 10),
(0, 255, 0), 2)
cv2.rectangle(
img,
(pose.keypoints[7][0] - 10, pose.keypoints[7][1] - 10),
(pose.keypoints[7][0] + 10, pose.keypoints[7][1] + 10),
(0, 255, 0), 2)
if bool(previous_poses):
prev_pose = previous_poses[0]
prep_displacement += (
(pose.keypoints[4][0] - prev_pose.keypoints[4][0])**2 +
(pose.keypoints[4][1] - prev_pose.keypoints[4][1])**
2)**0.5
prep_displacement += (
(pose.keypoints[7][0] - prev_pose.keypoints[7][0])**2 +
(pose.keypoints[7][1] - prev_pose.keypoints[7][1])**
2)**0.5
prep_time += (cv2.getTickCount() -
tik1) / cv2.getTickFrequency()
previous_poses = current_poses
current_time = (cv2.getTickCount() - tik1) / cv2.getTickFrequency()
video_time += current_time
cv2.putText(img, "displacement: %d" % int(prep_displacement), (20, 20),
cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)
cv2.putText(img, "prep time (s): %.1f" % (prep_time / 2), (20, 40),
cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)
cv2.putText(img, "video time (s): %.1f" % (video_time / 2), (20, 60),
cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1)
if mean_time == 0:
mean_time = current_time
else:
mean_time = mean_time * 0.95 + current_time * 0.05
#cv2.putText(img, 'FPS: {}'.format(int(1 / mean_time * 10) / 10),
# (40, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255))
cv2.imshow('Chicken Prep Test Demo', img)
key = cv2.waitKey(delay)
if key == 27: # esc
return
elif key == 112: # 'p'
if delay == 1:
delay = 0
else:
delay = 1
def run_demo(net, height_size, cpu, track, smooth, image_provider=None, use_realsense_cam=False, openvino=False):
if not image_provider and not use_realsense_cam:
raise ValueError('Either `image_provider` or `use_realsense_cam` must be provided')
if use_realsense_cam:
pipeline = init_realsense()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 33
while True:
# get frame from source provided
if use_realsense_cam:
frames = pipeline.wait_for_frames()
color_frame = frames.get_color_frame()
img = np.asanyarray(color_frame.get_data())
else:
img = next(image_provider)
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu, openvino=openvino)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(delay)
if key == 27: # esc
return
elif key == 112: # 'p'
if delay == 33:
delay = 0
else:
delay = 33
def run_demo(net, image_provider, height_size, cpu, track, smooth, com):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts # +1 for Hidden COM
previous_poses = []
# original delay 33
# 0 = pause / wait for input indefinetly
delay = 0
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones(
(num_keypoints + 1, 2), dtype=np.int32) * -1 # +1 here for COM
found_kpts = []
C_pts = []
BOS = [[-1, -1], [-1, -1]]
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
found_kpts.append(kpt_id)
# print(kpt_id, pose_keypoints[kpt_id], Pose.kpt_names[kpt_id]) # ====== HOLY GRAIL =========
if com:
COM, C_pts, BOS = compute_com(found_kpts, pose_keypoints)
pose_keypoints[-1] = COM
pose = Pose(pose_keypoints, pose_entries[n][18], C_pts, BOS)
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(delay)
if key == 27: # esc
return
elif key == 112: # 'p'
if delay == 33:
delay = 0
else:
delay = 33
def run_demo(net, image_provider, output_dir, height_size, cpu, track, smooth, generate_video):
net = net.eval()
if not cpu:
net = net.cuda()
if generate_video:
video_output_path = os.path.join(output_dir, output_dir.split("/")[-1] +"-annotations" + ".mp4")
#(1080, 1920, 3)
video_out = cv2.VideoWriter(video_output_path, cv2.VideoWriter_fourcc('m','p','4','v'), 10, (1920,1080))
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 33
for idx, img in enumerate(image_provider):
orig_img = img.copy()
frame_file = os.path.join(output_dir, output_dir.split("/")[-1]+f"_{idx:06}.txt")
with open(frame_file, "w") as frame_f:
print("Input the model is", orig_img.shape)
#print(output_path)
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
annotations_hand = []
annotations_head = []
for pose in current_poses:
pose.draw(img)
print(pose.hands_bbox, pose.head_bbox)
for hand_bbox in pose.hands_bbox:
x_center = (hand_bbox[0][0] + hand_bbox[1][0]) / (2*1920)
y_center = (hand_bbox[0][1] + hand_bbox[1][1]) / (2*1080)
x_offset = (hand_bbox[1][0] - hand_bbox[0][0]) / 1920
y_offset = (hand_bbox[1][1] - hand_bbox[0][1]) / 1080
if x_center < 0.0:
x_offset = x_offset - x_center
x_center = 0
hand_bbox_scaled = (x_center, y_center, x_offset, y_offset)
print(hand_bbox_scaled)
frame_f.write("8 " + ' '.join(map(str, hand_bbox_scaled)) + "\n")
annotations_hand.append(hand_bbox_scaled)
x_center = (pose.head_bbox[0][0] + pose.head_bbox[1][0]) / (2*1920)
y_center = (pose.head_bbox[0][1] + pose.head_bbox[1][1]) / (2*1080)
x_offset = (pose.head_bbox[1][0] - pose.head_bbox[0][0]) / 1920
y_offset = (pose.head_bbox[1][1] - pose.head_bbox[0][1]) / 1080
if x_center < 0.0:
x_offset = x_offset - x_center
x_center = 0
head_bbox_scaled = (x_center, y_center, x_offset, y_offset)
print(head_bbox_scaled)
frame_f.write("9 " + ' '.join(map(str, head_bbox_scaled)) + "\n")
annotations_head.append(head_bbox_scaled)
img = cv2.addWeighted(orig_img, 0.3, img, 0.7, 0)
for pose in current_poses:
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
img_filename = os.path.join(output_dir, output_dir.split("/")[-1] + f"_{idx:06}.jpg")
print(img_filename)
cv2.imwrite(img_filename, orig_img)
if generate_video:
video_out.write(img)
key = cv2.waitKey(delay)
if key == 27: # esc
return
elif key == 112: # 'p'
if delay == 33:
delay = 0
else:
delay = 33
if video_out:
video_out.release()
def evaluate(labels,
output_name,
images_folder,
net,
multiscale=False,
visualize=False):
net = net.cuda().eval()
base_height = 368
scales = [1]
if multiscale:
scales = [0.5, 1.0, 1.5, 2.0]
stride = 8
dataset = CocoValDataset(labels, images_folder)
coco_result = []
for sample in dataset:
file_name = sample["file_name"]
img = sample["img"]
avg_heatmaps, avg_pafs = infer(net, img, scales, base_height, stride)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(18): # 19th for bg
total_keypoints_num += extract_keypoints(
avg_heatmaps[:, :, kpt_idx], all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
avg_pafs)
coco_keypoints, scores = convert_to_coco_format(
pose_entries, all_keypoints)
image_id = int(file_name[0:file_name.rfind(".")])
for idx in range(len(coco_keypoints)):
coco_result.append({
"image_id": image_id,
"category_id": 1, # person
"keypoints": coco_keypoints[idx],
"score": scores[idx],
})
if visualize:
for keypoints in coco_keypoints:
for idx in range(len(keypoints) // 3):
cv2.circle(
img,
(int(keypoints[idx * 3]), int(keypoints[idx * 3 + 1])),
3,
(255, 0, 255),
-1,
)
cv2.imshow("keypoints", img)
key = cv2.waitKey()
if key == 27: # esc
return
with open(output_name, "w") as f:
json.dump(coco_result, f, indent=4)
run_coco_eval(labels, output_name)
def run_demo(net, action_net, image_provider, height_size, cpu):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
i = 0
for img in image_provider:
orig_img = img.copy()
# print(i)
if i % 1 == 0:
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size,
stride, upsample_ratio,
cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(
heatmaps[:, :, kpt_idx], all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(
all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones(
(num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
if len(pose.getKeyPoints()) >= 10:
current_poses.append(pose)
# current_poses.append(pose)
for pose in current_poses:
pose.img_pose = pose.draw(img, show_draw=True)
crown_proportion = pose.bbox[2] / pose.bbox[3] #宽高比
pose = action_detect(action_net, pose, crown_proportion)
if pose.pose_action == 'fall':
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2],
pose.bbox[1] + pose.bbox[3]), (0, 0, 255),
thickness=3)
cv2.putText(img, 'state: {}'.format(pose.pose_action),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
else:
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2],
pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
cv2.putText(img, 'state: {}'.format(pose.pose_action),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0))
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
cv2.waitKey(1)
i += 1
cv2.destroyAllWindows()
def run_lightweight(net,
img,
height_size=256,
cpu=False,
track=False,
smooth=True):
ts1 = time.time()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 1
orig_img = img.copy()
ts_netw1 = time.time()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
ts_netw2 = time.time()
# print("network time", (ts_netw2 - ts_netw1) * 1000, "ms")
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id,
0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio
- pad[1]) / scale
all_keypoints[kpt_id,
1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio
- pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
# pose.draw(img)
pose.draw_only_relevant(img)
#img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
#relevant only for demonstration
# cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
# (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
ts2 = time.time()
cv2.putText(img, '%.2fms' % ((ts2 - ts1) * 1000.0), (10, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))
return img
def run_demo(net, image_provider, height_size, cpu):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
color = [255, 0, 0] #To show the bones
colorcircle = [0, 0, 255] #To show the joints
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(18): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
global imagename
imagepath = imagename
if imagepath != "":
outputname = (imagepath[0].split('/')[-1])
outputname = "output/" + (outputname.split('.')[0]) + ".skl"
skeletonfile = open(outputname, "w")
skeletonfile.write(imagepath[0] + "\n")
# Write all the keypoints to output
for kpt in all_keypoints:
skeletonfile.write(str(kpt[0]) + "," + str(kpt[1]) + ",0.0\n")
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
######### calculate root point (since the prediction will give a set of joints that doesn't have root node)
bneck = 0.406394346548266
bleft = 0.295839233078765
bright = 0.297766420372969
neckx, necky = all_keypoints[int(pose_entries[n][1]),
0:2] #neck vertex
leftx, lefty = all_keypoints[int(pose_entries[n][11]),
0:2] #Left leg vertex
rightx, righty = all_keypoints[int(pose_entries[n][8]),
0:2] #right leg vertex
rootx = bneck * neckx + bleft * leftx + bright * rightx
rooty = bneck * necky + bleft * lefty + bright * righty
# Add root point to keypoints
skeletonfile.write(str(rootx) + "," + str(rooty) + ",0.0\n")
########### Plot root point
cv2.circle(img, (int(rootx), int(rooty)), 8, colorcircle, -1)
############ Draw lines
writeedges = []
for part_id in range(len(BODY_PARTS_PAF_IDS) - 2):
kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0]
global_kpt_a_id = pose_entries[n][kpt_a_id]
if global_kpt_a_id != -1:
x_a, y_a = all_keypoints[int(global_kpt_a_id), 0:2]
cv2.circle(img, (int(x_a), int(y_a)), 8, colorcircle, -1)
kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1]
global_kpt_b_id = pose_entries[n][kpt_b_id]
if global_kpt_b_id != -1:
x_b, y_b = all_keypoints[int(global_kpt_b_id), 0:2]
cv2.circle(img, (int(x_b), int(y_b)), 8, colorcircle, -1)
keypointsize = len(all_keypoints)
if global_kpt_a_id != -1 and global_kpt_b_id != -1:
if kpt_a_id == 1 and (kpt_b_id == 8 or kpt_b_id == 11):
if (global_kpt_a_id, keypointsize) not in writeedges:
writeedges.append((global_kpt_a_id, keypointsize))
if (keypointsize, global_kpt_b_id) not in writeedges:
writeedges.append((keypointsize, global_kpt_b_id))
# skeletonfile.write(str(global_kpt_a_id)+","+str(keypointsize)+"\n")
# skeletonfile.write(str(keypointsize)+","+str(global_kpt_b_id)+"\n")
cv2.line(img, (int(x_a), int(y_a)),
(int(rootx), int(rooty)), color, 3)
cv2.line(img, (int(rootx), int(rooty)),
(int(x_b), int(y_b)), color, 3)
else:
if (global_kpt_a_id,
global_kpt_b_id) not in writeedges:
writeedges.append(
(global_kpt_a_id, global_kpt_b_id))
# skeletonfile.write(str(global_kpt_a_id)+","+str(global_kpt_b_id)+"\n")
cv2.line(img, (int(x_a), int(y_a)),
(int(x_b), int(y_b)), color, 3)
# Writing the edges to file
for eachi in writeedges:
skeletonfile.write(str(eachi[0]) + "," + str(eachi[1]) + "\n")
skeletonfile.close()
if imagename != "":
imagename = (imagename[0].split('/')[-1])
imagename = "output/" + imagename
# print(imagename)
# cv2.imwrite(imagename, img)
else:
imagename = "output/tempop.png"
cv2.imwrite(imagename, img)
def postprocess(self, orig_image, output_dict):
"""
Reformat output into standard annotations
"""
heatmaps = output_dict['heatmaps']
pafs = output_dict['pafs']
scale = output_dict['scale']
pad = output_dict['pad']
num_keypoints = Pose.num_kpts
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id,
0] = (all_keypoints[kpt_id, 0] * self.stride /
self.upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id,
1] = (all_keypoints[kpt_id, 1] * self.stride /
self.upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
annotations = []
for pose_idx, pose in enumerate(current_poses):
person_label = 'person_{}'.format(pose_idx)
if envars.OUTPUT_POSES():
point_arr = []
found_points = set([])
for idx in range(pose.keypoints.shape[0]):
x = int(pose.keypoints[idx, 0])
y = int(pose.keypoints[idx, 1])
label = self.kpt_names[idx]
if x >= 0 and y >= 0:
point_arr.append({'x': x, 'y': y, 'label': label})
found_points.add(label)
links = [{
'from': x[0],
'to': x[1]
} for x in self.connections
if x[0] in found_points and x[1] in found_points]
annotations.append({
'kind': 'lines',
'label': person_label,
'confidence': pose.confidence,
'points': point_arr,
'links': links
})
if envars.OUTPUT_BOXES():
bbox_annotation = {
'kind': 'box',
'x': pose.bbox[0],
'y': pose.bbox[1],
'height': pose.bbox[3],
'width': pose.bbox[2],
'label': person_label,
'confidence': pose.confidence
}
annotations.append(bbox_annotation)
return {'name': self.name, 'annotations': annotations}
def run_demo(net, image_provider, height_size, cpu, track_ids):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = num_keys
previous_poses = []
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][num_keypoints])
current_poses.append(pose)
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
if track_ids == True:
propagate_ids(previous_poses, current_poses)
previous_poses = current_poses
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.3, (0, 0, 255))
allkey = True
landmarks = []
if (len(pose_entries) > 0):
for g in range(4):
if pose_entries[0][g] == -1:
allkey = False
if allkey:
for l in range(4):
landmarks.append(
int(all_keypoints[int(pose_entries[0][l])][0]))
landmarks.append(
int(all_keypoints[int(pose_entries[0][l])][1]))
if len(landmarks) > 0:
imgpts, modelpts, rotate_degree, center = face_orientation(
img, landmarks)
cv2.line(img, center, tuple(imgpts[1].ravel()), (0, 255, 0),
3) #GREEN
cv2.line(img, center, tuple(imgpts[0].ravel()), (
255,
0,
), 3) #BLUE
cv2.line(img, center, tuple(imgpts[2].ravel()), (0, 0, 255),
3) #RED
for j in range(len(rotate_degree)):
cv2.putText(img, ('{:05.2f}').format(float(rotate_degree[j])),
(10, 30 + (50 * j)),
cv2.FONT_HERSHEY_SIMPLEX,
1, (0, 255, 0),
thickness=2,
lineType=2)
cv2.imwrite('/home/charles/DockerData/Mattia/pose/poseFINAL/out.jpg',
img)
#cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(33)
if key == 27: # esc
return
def anime_frame(rgb, env, size=None, useSigmod=False, useTwice=False):
if env is None:
# return init_pose("./checkpoint_iter_370000.pth")
# return init_pose("./default_checkpoints/R.pth")
return init_pose("./refine4_checkpoints/checkpoint_iter_14000.pth")
net, previous_poses = env
stride = 8
upsample_ratio = 4
heatmaps, pafs, scale, pad = infer_fast(net, rgb, 368, stride, upsample_ratio)
num_keypoints = Pose.num_kpts
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[
1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[
0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
# if track:
track_poses(previous_poses, current_poses, smooth=True)
previous_poses = current_poses
env[1] = previous_poses
# return rgb, env
# print(rgb.min(), rgb.max())
# img = rgb.squeeze(0).permute(1, 2, 0).cpu().numpy()[:, :, ::-1]# * 255
# img = img.squeeze(0).permute(1, 2, 0).cpu().numpy()[:, :, ::-1]# * 255
# img += 0.5
# img *= 255
# img = img.astype(np.uint8)
img = np.zeros((rgb.shape[2], rgb.shape[3], rgb.shape[1]), dtype=np.uint8)
show_info = True
for pose in current_poses:
pose.draw(img, show_info)
show_info = False
# break
# for pose in current_poses:
# cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
# (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
# if track:
# cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16),
# cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
img = img[:, :, ::-1] #/ 255
# print(img.shape, img.dtype, img.min(), img.max())
img = torch.FloatTensor(img.astype(np.float32))
img /= 255
img = img.permute(2, 0, 1).unsqueeze(0).cuda()
output = rgb * 0.6 + img * 0.4
# output = img
return output, env
def run_demo(net, image_provider, height_size, cpu, track_ids): # , filename):
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts # 18개
previous_poses = []
c = 0
ttt = 0
idxx = 0
csv_dict = {
'frame_number': [],
'driver_index': [],
'is_driver_flag': []
} #,'state' : []}
driver_find_flag = False
weird_state_flag = False
extractor = Extractor('default_checkpoints/ckpt.t7', True)
find_class = Find_assault(extractor)
for img in image_provider:
is_driver_flag = False
t5 = time.time()
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]): ##사이즈 변환
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
# 운전자를 못 찾았으면 find_driver 에 들어감.
if driver_find_flag is False:
driver_find_flag, find_driver_count, find_state = find_class.find_driver(
current_poses, orig_img)
cv2.putText(img, "Driver_find_count : " + str(find_driver_count),
(0, 20), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0))
cv2.putText(img, "State : Find_Driver", (0, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0))
else:
# print("idxx : ", idxx)
is_driver_flag, driver_index, weird_state_count, weird_state_flag = find_class.is_driver(
current_poses, orig_img)
cv2.putText(img, "Weird_State_Count : " + str(weird_state_count),
(0, 20), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0))
if weird_state_flag:
cv2.putText(img, 'State : ABNORMAL', (0, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255))
else:
cv2.putText(img, "State : Driver_Found", (0, 50),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0))
# print("Driver_index :", driver_index)
# print("Driver_Flag : ", is_driver_flag)
# csv_dict['frame_number'].append(idxx)
# csv_dict['driver_index'].append(driver_index)
# csv_dict['is_driver_flag'].append(is_driver_flag)
# csv_dict['state'].append(state)
if track_ids == True: ##Track Poses
propagate_ids(previous_poses, current_poses)
previous_poses = current_poses
index_counter = 0
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
if is_driver_flag and index_counter == driver_index:
cv2.putText(img, 'DRIVER',
(pose.bbox[0] + 100, pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255))
index_counter += 1
tt = time.time()
fps = 1 / (tt - ttt)
print('fps=', fps)
ttt = time.time()
str_ = "FPS : %0.2f" % fps
cv2.putText(img, str_, (0, 100), cv2.FONT_HERSHEY_SIMPLEX, 1,
(0, 255, 0))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
# cv2_imshow(img)
cv2.imwrite('output/two_2/' + str(idxx) + '.png', img)
idxx += 1
key = cv2.waitKey(1)
if key == 27: # esc
return
df = pd.DataFrame(csv_dict)
def callback(self, data):
try:
cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8")
except CvBridgeError as e:
print(e)
## Rescale Image size
rescale_factor = 1
width = int(cv_image.shape[1] * rescale_factor)
height = int(cv_image.shape[0] * rescale_factor)
dim = (width, height)
resized_img = cv2.resize(cv_image, dim)
net = PoseEstimationWithMobileNet()
checkpoint = torch.load(
"/home/zheng/lightweight-human-pose-estimation.pytorch/checkpoint_iter_370000.pth",
map_location='cpu')
load_state(net, checkpoint)
height_size = 256
net = net.eval()
net = net.cuda()
net.eval()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 33
# img = cv2.imread("/home/zheng/lightweight-human-pose-estimation.pytorch/data/image_1400.jpg")
img = asarray(cv_image)
orig_img = img
heatmaps, pafs, scale, pad = infer_fast(net,
img,
height_size,
stride,
upsample_ratio,
cpu="store_true")
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
# cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
self.image_pub.publish(self.bridge.cv2_to_imgmsg(img, "bgr8"))
# cv2.imwrite('/home/zheng/Bureau/image_1400_key.jpg',img)
cv2.waitKey(2)
def run_demo(net, image_provider, height_size, cpu, track, smooth):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 33
dict_id_color = {}
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
dict_id_color_r = {}
for id_ in dict_id_color.keys():
flag_track = False
for pose in current_poses:
if id_ == pose.id:
flag_track = True
break
if flag_track:
dict_id_color_r[pose.id] = dict_id_color[pose.id]
dict_id_color = dict_id_color_r
track_thr = 0
for pose in current_poses:
# print('pose.id : ',pose.id)
if pose.id not in dict_id_color.keys():
R_ = random.randint(30,255)
G_ = random.randint(30,255)
B_ = random.randint(30,255)
dict_id_color[pose.id] = [[B_,G_,R_],1]
else:
dict_id_color[pose.id][1] += 1
if dict_id_color[pose.id][1]>track_thr:
pose.draw(img,color_x = dict_id_color[pose.id][0])
img = cv2.addWeighted(orig_img, 0.3, img, 0.7, 0)
for pose in current_poses:
if dict_id_color[pose.id][1]>track_thr:
cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0),4)
cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.namedWindow('Lightweight Human Pose Estimation Python Demo', 0)
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(delay)
if key == 27: # esc
return
elif key == 112: # 'p'
if delay == 33:
delay = 0
else:
delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = num_keys
previous_poses = []
for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][num_keypoints])
current_poses.append(pose)
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
if track_ids == True:
propagate_ids(previous_poses, current_poses)
previous_poses = current_poses
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
if save:
cv2.imwrite(
'/Users/Utente/Desktop/POSE-ESTIMATION/custom_pose/poseFINAL/out.jpg',
img)
else:
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(33)
if key == 27: # esc
return
def pose2d_detectHuman(net,
img,
height_size=256,
track=1,
smooth=1,
bVis=True):
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 33
if True:
# for img in image_provider:
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net,
img,
height_size,
stride,
upsample_ratio,
cpu=0)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if bVis:
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(delay)
if key == 27: # esc
return
elif key == 112: # 'p'
if delay == 33:
delay = 0
else:
delay = 33
return current_poses
def run_demo(net, height_size, track, smooth, record_vid, camera_type):
net = net.eval()
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
##Tarit defined
slope_threshold = 0.4
ear_slope_threshold = 0.5
eye_ear_slope_threshold = 0.5
not_detected = (-1, -1)
sleep_confirmation_time = 60 #in seconds
#flags to detect whether the person is sleeping or not
sleeping = False
timer_started = False
time_notified = 0
selected_pose = None
while True:
#msg, frame = receiver.receive(timeout = 60.0)
#img = cv2.imdecode(np.frombuffer(frame, dtype='uint8'), -1)
img = cap.read()
if camera_type == "jetson":
img = img[1300:1780, 1320:1960]
#start_time = time.time()
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
'''for pose in current_poses:
pose.draw(img)'''
##find longest_nect_to_nose_dst and select that pose
longest_nect_to_nose_dst = 0
for pose in current_poses:
nose = tuple(pose.keypoints[0])
neck = tuple(pose.keypoints[1])
##pythagoras
nect_to_nose_dst = pow((pow(abs(nose[0] - neck[0]), 2)) +
(pow(abs(nose[1] - neck[1]), 2)), 1 / 2)
if nect_to_nose_dst > longest_nect_to_nose_dst:
longest_nect_to_nose_dst = nect_to_nose_dst
selected_pose = pose
if selected_pose is not None:
selected_pose.draw(img)
nose = tuple(selected_pose.keypoints[0])
neck = tuple(selected_pose.keypoints[1])
l_ear = tuple(selected_pose.keypoints[16])
r_ear = tuple(selected_pose.keypoints[17])
l_eye = tuple(selected_pose.keypoints[15])
r_eye = tuple(selected_pose.keypoints[14])
#print(cal_slope(l_eye,l_ear),cal_slope(r_eye,r_ear))
##detect if the person back if facing to the camera
if nose == (-1, -1):
if l_ear != not_detected and r_ear != not_detected:
ear_slope = abs(l_ear[1] - r_ear[1]) / abs(l_ear[0] -
r_ear[0])
cv2.circle(img, l_ear, 5, (255, 0, 0), 3)
cv2.circle(img, r_ear, 5, (0, 255, 0), 3)
if ear_slope > ear_slope_threshold:
sleeping = True
print("sleeping")
else:
sleeping = False
else:
##out of condition, can't detect
sleeping = False
else:
cv2.circle(img, nose, 5, (255, 0, 0), 3)
cv2.circle(img, neck, 5, (0, 255, 0), 3)
slope_inverse = (nose[0] - neck[0]) / (nose[1] - neck[1])
l_ear_eye_slope = cal_slope(l_eye, l_ear)
r_ear_eye_slope = cal_slope(r_eye, r_ear)
#increase the slope_threshold if the person is turning their head
#print(pose.keypoints[16],pose.keypoints[17]) #print ear location
if l_ear == (-1, -1) or r_ear == (-1, -1):
slope_threshold = 1
print("one ear missing , Increasing slope_threshold")
else:
slope_threshold = 0.4
if abs(slope_inverse) > slope_threshold:
#cv2.putText(img,"".join([str(pose.id),"sleeping"]),(20,50),cv2.FONT_HERSHEY_COMPLEX,2,(255,0,0),3)
print("Sleeping (neck bend more than threshold)")
#cv2.putText(img,"sleeping",(20,50),cv2.FONT_HERSHEY_COMPLEX,2,(255,0,0),3)
sleeping = True
elif l_eye == not_detected or r_eye == not_detected:
sleeping = True
print("Sleeping (not seeing both eyes)")
elif l_ear_eye_slope < -0.6 or r_ear_eye_slope > 0.6 or l_ear_eye_slope > eye_ear_slope_threshold or r_ear_eye_slope < -eye_ear_slope_threshold:
sleeping = True
print("Sleeping (ears higher/lower than eyes)")
else:
print("Not sleeping")
sleeping = False
if sleeping:
if not timer_started:
t_start_sleep = time.time()
timer_started = True
else:
if time.time() - t_start_sleep > sleep_confirmation_time:
print("sending line message")
pic_name = "".join(
["log_data/",
str(time_notified), ".jpg"])
cv2.imwrite(pic_name, img)
#lineNotify("Elderly sleeping %d"%time_notified)
notifyFile("Elderly sleeping %d" % time_notified,
pic_name)
time_notified += 1
timer_started = False
sleeping = False
else:
timer_started = False
#song = AudioSegment.from_mp3("Alarm_Clock_Sound.mp3")
#play(song)
img = cv2.addWeighted(orig_img, 0.6, img, 0.6, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.imshow('Sleep detector', img)
if record_vid:
out_raw.write(orig_img)
out_pose.write(img)
#print((1/(time.time()-start_time)))
if cv2.waitKey(1) == 27: # esc
#receiver.close()
cap.stop()
if record_vid:
out_raw.release()
out_pose.release()
return
def socketComm(conn, index):
global terminate
global net
width = struct.unpack("<I", conn.recv(4))[0]
height = struct.unpack("<I", conn.recv(4))[0]
conn.settimeout(60)
print("Recieved webcam:", width, height)
if (width >= 3200) or (height >= 2400):
print("Too large, closing")
conn.close()
terminate.put(index)
return
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
delay = 33
if True:
#try:
while True:
datalen = conn.recv(4) #receives length of data before data
if len(datalen) == 0:
break
datalen = struct.unpack(
"<I", datalen)[0] #convert little Endian binary to integer
imageData = bytearray(
0) #master bytearray (imageData) to append to
while len(imageData) != datalen:
imageData.extend(conn.recv(datalen - len(imageData)))
imageData = np.frombuffer(imageData, dtype=np.uint8)
#decodes binary from jpeg into an image we can work with
img = cv2.imdecode(imageData, cv2.IMREAD_COLOR)
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, args.height_size,
stride, upsample_ratio,
args.cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(
heatmaps[:, :, kpt_idx], all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(
all_keypoints_by_type, pafs, demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones(
(num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
response = struct.pack("<36i", *pose_keypoints.flatten())
conn.send(struct.pack("<I", len(response)))
conn.send(response)
#except Exception as e:
# print(e)
conn.close()
print("Closing", index)
terminate.put(index)
def run_demo(net, image_provider, height_size, cpu):
net = net.eval()
net = net.cuda()
stride = 8
upsample_ratio = 4
color = [0, 0, 255]
global socket
cv2.namedWindow("Human Pose", cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty("Human Pose", cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
while True:
grabbed, img = image_provider.read()
orig_img = img.copy()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(18): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
humans = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
human = {}
#print(all_keypoints[n])
#all_keypoints[n] = savgol_filter(all_keypoints[n], 9, 3)
for part_id in range(17):
kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0]
global_kpt_a_id = pose_entries[n][kpt_a_id]
if global_kpt_a_id != -1:
x_a, y_a = all_keypoints[int(global_kpt_a_id), 0:2]
cv2.circle(img, (int(x_a), int(y_a)), 8, color, -1)
human[BODY_PARTS_IDS[int(kpt_a_id)]] = {
'position': {
'x': x_a,
'y': y_a
}
}
kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1]
global_kpt_b_id = pose_entries[n][kpt_b_id]
if global_kpt_b_id != -1:
x_b, y_b = all_keypoints[int(global_kpt_b_id), 0:2]
cv2.circle(img, (int(x_b), int(y_b)), 8, color, -1)
human[BODY_PARTS_IDS[int(kpt_b_id)]] = {
'position': {
'x': x_b,
'y': y_b
}
}
if global_kpt_a_id != -1 and global_kpt_b_id != -1:
cv2.line(img, (int(x_a), int(y_a)), (int(x_b), int(y_b)),
color, 2)
if any(human):
humans.append(human)
#socket.send_string(json.dumps(humans))
img = cv2.addWeighted(orig_img, 0.8, img, 0.8, 0)
cv2.imshow('Human Pose', img)
key = cv2.waitKey(1)
if key == 27: # esc
return
def run(net, image_provider, height_size, cpu, track, smooth):
ts1 = time.time()
net = setup(net, cpu)
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 1
ts2 = time.time()
print("setup took ", ts2 - ts1, " s")
print(ts2)
print(ts1)
for img in image_provider:
ts1 = time.time()
orig_img = img.copy()
print('Original Dimensions : ', orig_img.shape)
ts_netw1 = time.time()
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
ts_netw2 = time.time()
print("network time", (ts_netw2 - ts_netw1) * 1000, "ms")
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints): # 19th for bg
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n in range(len(pose_entries)):
if len(pose_entries[n]) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entries[n][kpt_id] != -1.0: # keypoint was found
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entries[n][kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entries[n][18])
current_poses.append(pose)
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(0, 255, 0))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
ts2 = time.time()
cv2.putText(img, '%.2fms' % ((ts2 - ts1) * 1000.0), (10, 20),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255))
cv2.imshow('Lightweight Human Pose Estimation Python Demo', img)
key = cv2.waitKey(delay)
if key == 27: # esc
return
elif key == 112: # 'p'
if delay == 1:
delay = 0
else:
delay = 1
def run_inference(net,
image_provider,
height_size,
cpu,
track,
smooth,
no_display,
json_view=False):
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
num_keypoints = Pose.num_kpts
previous_poses = []
delay = 100
if isinstance(image_provider, ImageReader):
delay = 0
for img in image_provider:
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(num_keypoints):
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs,
demo=True)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
current_poses = []
for n, pose_entry in enumerate(pose_entries):
if len(pose_entry) == 0:
continue
pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1
for kpt_id in range(num_keypoints):
if pose_entry[kpt_id] != -1.0:
pose_keypoints[kpt_id, 0] = int(
all_keypoints[int(pose_entry[kpt_id]), 0])
pose_keypoints[kpt_id, 1] = int(
all_keypoints[int(pose_entry[kpt_id]), 1])
pose = Pose(pose_keypoints, pose_entry[18])
current_poses.append(pose)
if json_view == True:
return current_poses
if not no_display:
if track:
track_poses(previous_poses, current_poses, smooth=smooth)
previous_poses = current_poses
for pose in current_poses:
pose.draw(img)
for pose in current_poses:
cv2.rectangle(
img, (pose.bbox[0], pose.bbox[1]),
(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]),
(32, 202, 252))
if track:
cv2.putText(img, 'id: {}'.format(pose.id),
(pose.bbox[0], pose.bbox[1] - 16),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255))
cv2.imshow('PoseCamera', img)
key = cv2.waitKey(delay)
if key == 27:
return
def convert_to_skelets(in_, out_, cpu=False, height_size=256):
# height_size - network input layer height size
# cpu - True if we would like to run in CPU
print('start convert to skelets')
# mask that shows - this is bed
mask = cv2.imread(os.path.join('mask', 'mask.jpg'), 0)
mask = cv2.normalize(mask,
None,
alpha=0,
beta=1,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_32F)
net = PoseEstimationWithMobileNet()
load_state(net, checkpoint)
net = net.eval()
if not cpu:
net = net.cuda()
stride = 8
upsample_ratio = 4
max_number = 963
num_img = 0
stream = cv2.VideoCapture("rtsp://*****:*****@62.140.233.76:554")
# for num in range(0, max_number + 1):
while (True):
# frame = 'frame' + str(num) + '.jpg'
# img = cv2.imread(os.path.join(in_, frame), cv2.IMREAD_COLOR)
r, img = stream.read()
# cv2.destroyAllWindows()
# find the place of the bed - and add border to it, so we can cut the unnecessary part
# apply object detection and find bed
# output is an image with black pixels of not bed, and white pixels of bed
heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride,
upsample_ratio, cpu)
total_keypoints_num = 0
all_keypoints_by_type = []
for kpt_idx in range(18):
total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx],
all_keypoints_by_type,
total_keypoints_num)
pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type,
pafs)
for kpt_id in range(all_keypoints.shape[0]):
all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride /
upsample_ratio - pad[1]) / scale
all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride /
upsample_ratio - pad[0]) / scale
# how many persons in image
num_persons = len(pose_entries)
# num_img more than time_period - we delete first second and add the last second
bones_detected = np.zeros(len(bones_to_detect))
bones_xa = np.zeros(len(bones_to_detect))
bones_ya = np.zeros(len(bones_to_detect))
bones_xb = np.zeros(len(bones_to_detect))
bones_yb = np.zeros(len(bones_to_detect))
bones_in_bed = np.zeros(len(bones_to_detect))
for n in range(num_persons):
count_person_not_in_bed = 1
for id_x in range(len(bones_to_detect)):
bones_detected[id_x] = 0
bones_xa[id_x] = 0
bones_ya[id_x] = 0
bones_xb[id_x] = 0
bones_yb[id_x] = 0
bones_in_bed[id_x] = 0
if len(pose_entries[n]) == 0:
continue
for id_, part_id in enumerate(bones_to_detect):
kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0]
global_kpt_a_id = pose_entries[n][kpt_a_id]
kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1]
global_kpt_b_id = pose_entries[n][kpt_b_id]
# if both points are detected
if global_kpt_a_id != -1 and global_kpt_b_id != -1:
bones_xa[id_], bones_ya[id_] = all_keypoints[
int(global_kpt_a_id), 0:2]
bones_xb[id_], bones_yb[id_] = all_keypoints[
int(global_kpt_b_id), 0:2]
if mask[int(bones_ya[id_])][int(
bones_xa[id_])] == 1 and mask[int(
bones_yb[id_])][int(bones_xb[id_])] == 1:
bones_in_bed[id_] = 1
bones_detected[id_] = 1
sum_bones = 0
for id_, val in enumerate(bones_in_bed):
sum_bones += val
if sum_bones == len(bones_in_bed):
# anomaly
# we take mean vector of 2 vectors of bones 6 and 9
bone_xa = (bones_xa[0] + bones_xa[2]) / 2
bone_ya = (bones_ya[0] + bones_ya[2]) / 2
bone_xb = (bones_xb[0] + bones_xb[2]) / 2
bone_yb = (bones_yb[0] + bones_yb[2]) / 2
x1 = bone_xb - bone_xa
y1 = bone_yb - bone_ya
x2 = 100
y2 = 0
global anomaly_checker
alfa = math.acos(
(x1 * x2 + y1 * y2) /
(math.sqrt(x1**2 + y1**2) * math.sqrt(x2**2 + y2**2)))
# if alfa is close to 90 degree - anomaly
if min(abs(alfa - rad_90), abs(alfa - rad_270)) <= threshold:
print('num_persons', num_persons)
if num_persons == 1:
anomaly_checker = np.delete(anomaly_checker, 0)
anomaly_checker = np.append(anomaly_checker, 1)
cv2.imwrite(os.path.join('out_out', frame), img)
if np.sum(anomaly_checker) >= SEC_WITHOUT_HELP:
print('ALARM!')
num_img += 1
if not os.path.exists(out_):
os.mkdir(out_)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
print('done convert to skelets')
