def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
if opt.matching:
preds = getMultiPeakPrediction(hm_data, pt1.numpy(),
pt2.numpy(),
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img,
preds_scores)
result = {'imgname': im_name, 'result': result}
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def predict(self, image, bboxs, bboxs_scores):
inps, pt1, pt2 = crop_dets(image, bboxs, self.inp_h, self.inp_w)
pose_hm = self.model(inps.to(self.device)).cpu().data
# Cut eyes and ears.
pose_hm = torch.cat([pose_hm[:, :1, ...], pose_hm[:, 5:, ...]], dim=1)
xy_hm, xy_img, scores = getPrediction(pose_hm, pt1, pt2, self.inp_h, self.inp_w,
pose_hm.shape[-2], pose_hm.shape[-1])
result = pose_nms(bboxs, bboxs_scores, xy_img, scores)
return result
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW, opt.nClasses)
# print('dataloader:',boxes.shape, scores.shape, preds_img.shape, preds_scores.shape)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {
'imgname': im_name,
'result': result,
'boxes':boxes
}
# print('dataloader.py:result:',result)
# print('dataloader.py:hm_data.shape:',hm_data.shape)
# print('dataloader.py:preds_hm.shape:',preds_hm.shape)
# self.count+=1
# print('dataloader.py:count:',self.count)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis and len(img)!=0:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img and len(img)!=0:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video and len(img)!=0:
self.stream.write(img)
else:
time.sleep(0.1)
def update(boxes, scores, hm_data, pt1, pt2, orig_img, i):
orig_img = np.array(orig_img, dtype=np.uint8)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(
boxes, scores, preds_img, preds_scores)
result = {
'imgname': str(i),
'result': result
}
img = vis_frame(orig_img, result)
return img, result
def fetch_result(boxes, scores, hm_data, pt1, pt2, ori_im, im_name):
ori_im = np.array(ori_im, dtype=np.uint8)
if boxes is None:
return None
preds_hm, preds_img, preds_scores = getPrediction(hm_data, pt1, pt2,
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {
'imgname': im_name,
'result': result,
'bbox': boxes,
}
return result
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
if boxes is None:
if opt.save_img or opt.save_video:
#img = display_frame(orig_img, result, opt.outputpath)
img = orig_img
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'imgname': im_name, 'result': result}
self.final_result.append(result)
if opt.save_img or opt.save_video:
#img = display_frame(orig_img, result, opt.outputpath)
img = vis_frame(orig_img, result)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.01)
def process(self, boxes, scores, hm_data, pt1, pt2, orig_img, im_name):
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
return orig_img, []
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
self.result = pose_nms(boxes, scores, preds_img, preds_scores)
if self.result:
result = self.locate()
# result = self.result
result = {'imgname': im_name, 'result': result}
self.final_result.append(result)
img_black, pred_black = vis_frame_black(orig_img, result)
self.img = img_black
self.skeleton = pred_black
img, pred = vis_frame(orig_img, result)
return img, self.skeleton, self.img
else:
return orig_img, [], orig_img
def gen_pose(self):
with torch.no_grad():
(inps, orig_img, boxes, scores, pt1,
pt2) = self.human_detect_result
if boxes is None or boxes.nelement() == 0:
self.det_human_num = 0
self.out_img = orig_img
self.result = None
self.resultNew = None
return
#print(inps, 'inps')
#print(type(boxes), 'boxes')
#
inps = inps.cuda()
hm = self.pose_model(inps)
hm = [hm]
hm = torch.cat(hm)
hm = hm.cpu()
hm_data = hm
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
self.det_human_num = 0
self.out_img = orig_img
self.result = None
self.resultNew = None
return
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
self.result = result
self.resultNew = conserv_result_format_to_list(self.result)
resultDict = {'imgname': 'zhjs', 'result': result}
self.out_img = vis_frame(orig_img, resultDict)
self.det_human_num = len(result)
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(self.outputpath, self.dir_folder,
im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
# if opt.matching:
# preds = getMultiPeakPrediction(
# hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
# result = matching(boxes, scores.numpy(), preds)
# else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
# (hm_data, pt1, pt2, 320, 256, 80, 64)
result = pose_nms(boxes, scores, preds_img, preds_scores)
# (bbox定位list, bbox评分list, 位姿定位list, 位姿评分list)
# result[ {'keypoints', 'kp_score', 'proposal_score'}, {---}, ...], pPose_nms.py line-114
result = {'imgname': im_name, 'result': result}
# self.final_result.append(result)
self.show_img = vis_frame(orig_img, result)
cv2.imwrite(
os.path.join(self.outputpath, self.dir_folder,
im_name), img)
# if opt.save_img or opt.save_video or opt.vis:
# img = vis_frame(orig_img, result)
# if opt.vis:
# cv2.imshow("AlphaPose Demo", img)
# cv2.waitKey(30)
# if opt.save_img:
# cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
# if opt.save_video:
# self.stream.write(img)
else:
# time.sleep(0.1)
pass
def update(self):
next_id = 0
car_next_id = 0
bbox_dets_list_list = []
keypoints_list_list = []
car_dets_list_list = []
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
start_time = getTime()
(boxes, scores, hm_data, pt1, pt2, orig_img, img_id, CAR) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is not None:
boxes = boxes.astype(np.int32)
img = orig_img
# text_filled2(img,(5,200),str(img_id),LIGHT_GREEN,2,2)
bbox_dets_list = [] # keyframe: start from empty
keypoints_list = [] # keyframe: start from empty
# print(boxes)
if boxes is None: # No person detection
pass
# bbox_det_dict = {"img_id": img_id,
# "det_id": 0,
# "track_id": None,
# "bbox": [0, 0, 2, 2]}
# bbox_dets_list.append(bbox_det_dict)
#
# keypoints_dict = {"img_id": img_id,
# "det_id": 0,
# "track_id": None,
# "keypoints": []}
# keypoints_list.append(keypoints_dict)
else:
if opt.matching:
preds = getMultiPeakPrediction(
hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(hm_data, pt1, pt2, opt.inputResH,
opt.inputResW, opt.outputResH,
opt.outputResW)
# print('number of result', preds_hm, preds_scores )
result = pose_nms(boxes, scores, preds_img, preds_scores) # list type
# result = { 'keypoints': , 'kp_score': , 'proposal_score': , 'bbox' }
if img_id > 0: # First frame does not have previous frame
bbox_list_prev_frame = bbox_dets_list_list[img_id - 1].copy()
keypoints_list_prev_frame = keypoints_list_list[img_id - 1].copy()
else:
bbox_list_prev_frame = []
keypoints_list_prev_frame = []
# boxes.size(0)
num_dets = len(result)
for bbox in boxes:
x, y, w, h = bbox.astype(np.uint32)
cv2.rectangle(orig_img, (x, y), (x + w, y + h), (253, 222, 111), 1)
for det_id in range(num_dets): # IOU tracking for detections in current frame.
# detections for current frame
# obtain bbox position and track id
result_box = result[det_id]
kp_score = result_box['kp_score']
proposal_score = result_box['proposal_score'].numpy()[0]
if proposal_score < 1.3:
continue
keypoints = result_box['keypoints'] # torch, (17,2)
keypoints_pf = np.zeros((15, 2))
idx_list = [16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9, 0, 0, 0]
for i, idx in enumerate(idx_list):
keypoints_pf[i] = keypoints[idx]
keypoints_pf[12] = (keypoints[5] + keypoints[6]) / 2 # neck
# COCO-order {0-nose 1-Leye 2-Reye 3-Lear 4Rear 5-Lsho 6-Rsho 7-Lelb 8-Relb 9-Lwri 10-Rwri 11-Lhip 12-Rhip 13-Lkne 14-Rkne 15-Lank 16-Rank}
# PoseFLow order #{0-Rank 1-Rkne 2-Rhip 3-Lhip 4-Lkne 5-Lank 6-Rwri 7-Relb 8-Rsho 9-Lsho 10-Lelb 11-Lwri 12-neck 13-nose 14-TopHead}
bbox_det = bbox_from_keypoints(keypoints) # xxyy
# bbox_in_xywh = enlarge_bbox(bbox_det, enlarge_scale)
# bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
# Keyframe: use provided bbox
# if bbox_invalid(bbox_det):
# track_id = None # this id means null
# keypoints = []
# bbox_det = [0, 0, 2, 2]
# # update current frame bbox
# bbox_det_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "bbox": bbox_det}
# bbox_dets_list.append(bbox_det_dict)
# # update current frame keypoints
# keypoints_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "keypoints": keypoints}
# keypoints_list.append(keypoints_dict)
# continue
# # update current frame bbox
if img_id == 0: # First frame, all ids are assigned automatically
track_id = next_id
next_id += 1
else:
track_id, match_index = get_track_id_SpatialConsistency(bbox_det, bbox_list_prev_frame)
# print('track' ,track_id, match_index)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
# update current frame bbox
bbox_det_dict = {"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"bbox": bbox_det}
# update current frame keypoints
keypoints_dict = {"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"keypoints": keypoints,
'kp_poseflow': keypoints_pf,
'kp_score': kp_score,
'bbox': bbox_det,
'proposal_score': proposal_score}
bbox_dets_list.append(bbox_det_dict)
keypoints_list.append(keypoints_dict)
num_dets = len(bbox_dets_list)
for det_id in range(num_dets): # if IOU tracking failed, run pose matching tracking.
bbox_det_dict = bbox_dets_list[det_id]
keypoints_dict = keypoints_list[det_id]
# assert (det_id == bbox_det_dict["det_id"])
# assert (det_id == keypoints_dict["det_id"])
if bbox_det_dict["track_id"] == -1: # this id means matching not found yet
# track_id = bbox_det_dict["track_id"]
track_id, match_index = get_track_id_SGCN(bbox_det_dict["bbox"], bbox_list_prev_frame,
keypoints_dict["kp_poseflow"],
keypoints_list_prev_frame)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
bbox_det_dict["track_id"] = track_id
keypoints_dict["track_id"] = track_id
# if still can not find a match from previous frame, then assign a new id
# if track_id == -1 and not bbox_invalid(bbox_det_dict["bbox"]):
if track_id == -1:
bbox_det_dict["track_id"] = next_id
keypoints_dict["track_id"] = next_id
next_id += 1
# update frame
# print('keypoint list', len(keypoints_list))
vis_frame(img, keypoints_list)
"""
Car
"""
if CAR is not None:
car_np = CAR
new_car_bboxs = car_np[:, 0:4].astype(np.uint32) # b/ x y w h c / cls_conf, cls_idx
new_car_score = car_np[:, 4]
cls_conf = car_np[:, 4]
# print("id: ", img_id , " ------------ " , new_car_bboxs, new_car_score)
# cls_conf = car_np[:, 6]
car_dest_list = []
if img_id > 1: # First frame does not have previous frame
car_bbox_list_prev_frame = car_dets_list_list[img_id - 1].copy()
else:
car_bbox_list_prev_frame = []
# print('car bbox list prev frame ', len(car_bbox_list_prev_frame))
for c, score, conf in zip(new_car_bboxs, new_car_score, cls_conf):
# car_bbox_det = c
# car_bbox_det = x1y1x2y2_to_xywh(c)
bbox_det = c
# bbox_in_xywh = enlarge_bbox(car_bbox_det, enlarge_scale)
# bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
if img_id == 0: # First frame, all ids are assigned automatically
car_track_id = car_next_id
car_next_id += 1
else:
car_track_id, match_index = get_track_id_SpatialConsistency(bbox_det,
car_bbox_list_prev_frame)
# print(car_track_id, match_index)
if car_track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del car_bbox_list_prev_frame[match_index]
bbox_det_dict = {"img_id": img_id,
"track_id": car_track_id,
"bbox": bbox_det,
"score": score,
"conf": conf}
car_dest_list.append(bbox_det_dict)
for car_bbox_det_dict in car_dest_list: # detections for current frame
if car_bbox_det_dict["track_id"] == -1: # this id means matching not found yet
car_bbox_det_dict["track_id"] = car_next_id
car_next_id += 1
self.tracking(car_dest_list)
car_dets_list_list.append(car_dest_list)
else:
car_dest_list = []
bbox_det_dict = {"img_id": img_id,
"det_id": 0,
"track_id": None,
"bbox": [0, 0, 2, 2],
"score": 0,
"conf": 0}
car_dest_list.append(bbox_det_dict)
car_dets_list_list.append(car_dest_list)
bbox_dets_list_list.append(bbox_dets_list)
keypoints_list_list.append(keypoints_list)
if img_id != 0:
self.car_person_detection(car_dest_list, bbox_dets_list, img)
self.car_parking_detection(car_dest_list, img, img_id)
ckpt_time, det_time = getTime(start_time)
cv2.putText(img, str(1 / det_time), (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(33)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
while True:
sys.stdout.flush()
print("generator len : " + str(self.Q.qsize()))
# if the thread indicator variable is set, stop the
# thread
# if self.stopped:
# cv2.destroyAllWindows()
# if self.save_video:
# self.stream.release()
# return
# otherwise, ensure the queue is not empty
if not self.det_processor.Q.empty():
with torch.no_grad():
(inps, orig_img, im_name, boxes, scores, pt1,
pt2) = self.det_processor.read()
if orig_img is None:
sys.stdout.flush()
print(f'{im_name} image read None: handle_video')
break
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None or boxes.nelement() == 0:
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = (None, None, None, None, None, orig_img,
im_name.split('/')[-1])
res = {'keypoints': -1, 'image': orig_img}
self.Q.put(res) #TODO
# cv2.imwrite("/home/hrs/Desktop/dd/now.jpg", orig_img)
# img = orig_img
# cv2.imshow("AlphaPose Demo", img)
# cv2.waitKey(30)
######################################################################################
# self.image = self.ax_in.imshow(orig_img, aspect='equal')
# self.image.set_data(orig_img)
# plt.draw()
# plt.pause(0.000000000000000001)
######################################################################################
# if opt.save_img or opt.save_video or opt.vis:
# img = orig_img
# if opt.vis:
# cv2.imshow("AlphaPose Demo", img)
# cv2.waitKey(30)
# if opt.save_img:
# cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
# if opt.save_video:
# self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
datalen = inps.size(0)
batchSize = 20 #args.posebatch()
leftover = 0
if datalen % batchSize:
leftover = 1
num_batches = datalen // batchSize + leftover
hm = []
# sys.stdout.flush()
# print("hhhh")
for j in range(num_batches):
inps_j = inps[j * batchSize:min(
(j + 1) * batchSize, datalen)].cuda()
hm_j = self.pose_model(inps_j)
hm.append(hm_j)
# time1 = time.time()
hm = torch.cat(hm)
hm = hm.cpu().data
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = (boxes, scores, hm, pt1, pt2, orig_img,
im_name.split('/')[-1])
if opt.matching:
preds = getMultiPeakPrediction(
hm_data, pt1.numpy(), pt2.numpy(),
opt.inputResH, opt.inputResW, opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH,
opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img,
preds_scores)
result = {'imgname': im_name, 'result': result}
self.final_result.append(result)
# time2 = time.time()
# print(time2-time1)
######################################################################################
# img = vis_frame(orig_img, result)
# cv2.imshow("AlphaPose Demo", img)
# cv2.imwrite("/home/hrs/Desktop/dd/now.jpg", img)
# cv2.waitKey(30)
########################################################################
# self.point.set_offsets(keypoints[self.i])
# self.image = self.ax_in.imshow(orig_img, aspect='equal')
# self.image.set_data(orig_img)
# plt.draw()
# plt.pause(0.000000000000000001)
##########################################################################
if not result['result']: # No people
res = {'keypoints': -1, 'image': orig_img}
self.Q.put(res) #TODO
else:
kpt = max(
result['result'],
key=lambda x: x['proposal_score'].data[0] *
calculate_area(x['keypoints']),
)['keypoints']
res = {'keypoints': kpt, 'image': orig_img}
self.Q.put(res)
# kpt_np = kpt.numpy()
# n = kpt_np.shape[0]
# print(kpt_np.shape)
# point_list = [(kpt_np[m, 0], kpt_np[m, 1]) for m in range(17)]
# for point in point_list:
# cv2.circle(pose_img, point, 1, (0, 43, 32), 4)
# cv2.imshow(self.window, pose_img)
# cv2.waitKey()
# if opt.save_img or opt.save_video or opt.vis:
# img = vis_frame(orig_img, result)
# if opt.vis:
# cv2.imshow("AlphaPose Demo", img)
# cv2.waitKey(30)
# if opt.save_img:
# cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
# if opt.save_video:
# self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
temp_kps = []
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
#print(len(result))
#print(boxes.shape)
result = []
if preds_img.shape[0] > 0:
for re in range(preds_img.shape[0]):
pos = preds_img[re].unsqueeze(0).numpy()
pos = self.aligner.align_points(pos)[0]
pos = (pos[..., :2] - 129) / 255
pos = torch.FloatTensor(pos)
kp = torch.cat(
(pos, preds_scores[re].unsqueeze(1)), 1)
kp = kp.unsqueeze(0)
kp = kp.reshape([1, -1]).cuda()
kp = kp.repeat(9, 1).reshape(1, -1)
outputs = self.pos_reg_model(kp)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result.append({
'class':
str(float(classidx)),
'keypoints':
preds_img[re],
'kp_score':
preds_scores[re].unsqueeze(1),
'bbox':
boxes[re]
})
# print(preds)
result = {'imgname': im_name, 'result': result}
# if len(result)>0:
# for re in range(len(result)):
# pos = result[re]['keypoints'].unsqueeze(0).numpy()
# pos = self.aligner.align_points(pos)[0]
# pos = (pos[..., :2] - 129) / 255
# pos = torch.FloatTensor(pos)
# kp = torch.cat((pos, result[0]['kp_score']), 1)
# kp = kp.unsqueeze(0)
# kp = kp.reshape([1, -1]).cuda()
# kp = kp.repeat(9, 1).reshape(1, -1)
# outputs = self.pos_reg_model(kp)
# _, preds = torch.max(outputs, 1)
# classidx = preds.cpu()
# result[re]['class'] = str(float(classidx))
# # print(preds)
# result = {
# 'imgname': im_name,
# 'result': result
# }
self.result_Q.put(result)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis',
im_name), img)
if opt.save_video:
self.stream.write(img)
# 发送图像
img = vis_frame(orig_img, result)
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
self.tcp_client.send_img(img)
else:
time.sleep(0.1)
def update(self):
count = 0
# filepath = '/home/yurik/Documents/Program/Alphapose_zed_video/testdata/20191014/walkstraightly/walkstraightly.svo'
# init = sl.InitParameters(svo_input_filename=filepath,svo_real_time_mode=False)
# init.depth_mode = sl.DEPTH_MODE.DEPTH_MODE_QUALITY
# cam = sl.Camera()
# runtime = sl.RuntimeParameters()
# status = cam.open(init)
# mat = sl.Mat()
# zeroarr = np.zeros((720,1280,3))
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
if opt.matching:
preds = getMultiPeakPrediction(
hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(
boxes, scores, preds_img, preds_scores)
result = {
'imgname': im_name,
'result': result
}
# 3D coordinates computation
ppl = result['result']
ppl_num = len(ppl)
self.coordinates_u, self.coordinates_v, self.truex, self.truey, self.dists = fl.people_3d_coord(ppl, ppl_num,
self.video_mode, self.camMtx1, orig_img)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
# err = cam.grab(runtime)
# if err == sl.ERROR_CODE.SUCCESS:
# cam.retrieve_image(mat, sl.VIEW.VIEW_DEPTH)
# depthmap = mat.get_data()
# if img.shape[2] == 3:
# depthmap = cv2.cvtColor(depthmap, cv2.COLOR_RGBA2RGB)
# depthmap = cv2.resize(depthmap, (int(img.shape[1]/2), img.shape[0]))
# depthmap = cv2.applyColorMap(depthmap, cv2.COLORMAP_JET)
# depthmap = np.hstack((depthmap, zeroarr))
# depthmap = depthmap.astype(np.uint8)
# img = cv2.addWeighted(img, 0.5, depthmap, 0.5, 3)
if len(self.coordinates_v) > 0 and len(self.coordinates_u) > 0:
for i in range(len(self.coordinates_v)):
# cv2.putText(img, 'z:' + str(round((self.dists[i] / 10), 1)),
# (int(self.coordinates_u[i]), int(self.coordinates_v[i]) - 15),
# cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 3, 8)
cv2.putText(img, str(round((self.truex[i] / 10), 1)),
(int(self.coordinates_u[i]), int(self.coordinates_v[i]) - 15),
cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 3, 8)
cv2.putText(img, str(round((self.truey[i] / 10), 1)),
(int(self.coordinates_u[i]) + 200, int(self.coordinates_v[i]) - 15),
cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 3, 8)
cv2.putText(img, str(round((self.dists[i] / 10), 1)),
(int(self.coordinates_u[i]) + 400, int(self.coordinates_v[i]) - 15),
cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 3, 8)
cv2.putText(img, 'frames: ' + str(count), (620, 620), cv2.FONT_HERSHEY_PLAIN, 2, (0,100,90), 3, 8)
else:
cv2.putText(img, '[N/A]',
(40, 620), cv2.FONT_HERSHEY_PLAIN, 2, (0, 100, 90), 3, 8)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
count = count + 1
def update(self):
# keep looping infinitely
while True:
sys.stdout.flush()
print("generator len : " + str(self.Q.qsize()))
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.det_processor.Q.empty():
with torch.no_grad():
(inps, orig_img, im_name, boxes, scores, pt1,
pt2) = self.det_processor.read()
if orig_img is None:
print(f'{im_name} image read None: handle_video')
break
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None or boxes.nelement() == 0:
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = (None, None, None, None, None, orig_img,
im_name.split('/')[-1])
# if opt.save_img or opt.save_video or opt.vis:
# img = orig_img
# if opt.vis:
# cv2.imshow("AlphaPose Demo", img)
# cv2.waitKey(30)
# if opt.save_img:
# cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
# if opt.save_video:
# self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
datalen = inps.size(0)
batchSize = 10 #args.posebatch()
leftover = 0
if datalen % batchSize:
leftover = 1
num_batches = datalen // batchSize + leftover
hm = []
sys.stdout.flush()
print("hhhh")
for j in range(num_batches):
inps_j = inps[j * batchSize:min(
(j + 1) * batchSize, datalen)] #.cuda()
hm_j = self.pose_model(inps_j)
hm.append(hm_j)
hm = torch.cat(hm)
hm = hm.cpu().data
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = (boxes, scores, hm, pt1, pt2, orig_img,
im_name.split('/')[-1])
if opt.matching:
preds = getMultiPeakPrediction(
hm_data, pt1.numpy(), pt2.numpy(),
opt.inputResH, opt.inputResW, opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH,
opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img,
preds_scores)
result = {'imgname': im_name, 'result': result}
self.final_result.append(result)
kpts = []
no_person = []
if not result['result']: # No people
self.Q.put(None) #TODO
else:
self.Q.put(result)
# if opt.save_img or opt.save_video or opt.vis:
# img = vis_frame(orig_img, result)
# if opt.vis:
# cv2.imshow("AlphaPose Demo", img)
# cv2.waitKey(30)
# if opt.save_img:
# cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
# if opt.save_video:
# self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {
'imgname': im_name,
'result': result
} # append imgname here.
# result here includes imgname, bbox, kps, kp_score, proposal_score
# Critical, run pnp algorithm here to get 6d pose.
# embed()
if result['result']:
kp_2d = np.array(result['result'][0]['keypoints'])
kp_3d = np.array(self.kp_3d)
R, t = pnp(kp_3d, kp_2d, self.cam_K)
result.update({'cam_R': R, 'cam_t': t})
else:
result.update({'cam_R': [], 'cam_t': []})
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
temp_kps=[]
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
pos = result[0]['keypoints'].unsqueeze(0).numpy()
pos = self.aligner.align_points(pos)[0]
pos = (pos[..., :2] - 129) / 255
pos = torch.FloatTensor(pos)
kp = torch.cat((pos, result[0]['kp_score']), 1)
kp = kp.unsqueeze(0)
if len(temp_kps) < 9:
kp = kp.reshape([1, -1]).cuda()
temp_kps.append(kp)
kp = kp.repeat(9, 1).reshape(1, -1)
outputs = self.pos_reg_model(kp)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
else:
kp = kp.cuda().reshape(1,-1)
temp_kps.append(kp)
temp_kps.pop(0)
_temp_kps = torch.cat(temp_kps)
_temp_kps.cuda()
_temp_kps = _temp_kps.reshape([1, -1])
outputs = self.pos_reg_model(_temp_kps)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
# print(preds)
result = {
'imgname': im_name,
'result': result
}
self.result_Q.put((boxes, classidx))
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# num = 1
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
img_T = get_frame(self.iter, self.Cap_T)
self.iter +=1
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
if self.iter<self.total_len:
img = orig_img
if opt.save_img or opt.save_video or opt.vis:
img_T = cv2.resize(img_T, (img.shape[1], img.shape[0]), interpolation = cv2.INTER_AREA)
img = np.hstack((img,img_T))
cv2.imshow("Action_Trainer Demo", img)
cv2.waitKey(60)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# img = orig_img
cv2.putText(img, "The movie repeats, press ctrl+ c key to terminate" , (50, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 3)
cv2.imshow("Action_Trainer Demo", img)
cv2.waitKey(300)
# self.Q.queue.clear()
self.iter=0
with self.Q.mutex:
self.Q.queue.clear()
# self.Q.queue.clear()
else:
preds_hm, preds_img, preds_scores = getPrediction(hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result2 = {
'imgname': im_name,
'result': result
}
self.final_result.append(result2)
try:
if self.iter< self.total_len:
k= self.pose_id_[self.iter]
if k==0:
continue
Old_pose = np.array(result[0]['keypoints'])
poseU = old2new_joint(Old_pose)
poseU_align = align_torso(poseU)
poseT = self.T_pose[k]
poseT_align = align_torso(poseT)
theta = get_diff( poseT_align, poseU_align)
img = draw_pose_final(poseT, poseU, orig_img, theta)
# img_T = draw_pos(poseT, img_T)
img_T = cv2.resize(img_T, (img.shape[1], img.shape[0]), interpolation = cv2.INTER_AREA)
img = np.hstack((img,img_T))
# cv2.putText(img, str(self.iter) , (900, 150), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 3)
cv2.imshow("Action_Trainer Demo", img)
cv2.waitKey(30)
else:
cv2.putText(img, "The movie repeats, press ctrl+c key to terminate" , (50, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 3)
cv2.imshow("Action_Trainer Demo", img)
cv2.waitKey(300)
self.iter=0
with self.Q.mutex:
self.Q.queue.clear()
#
except:
print("***********", self.iter)
else:
time.sleep(0.1)
def detect_main(args, im_names, yolo_model, pose_net):
# Load input images
data_loader = ImageLoader(im_names, batchSize=args.detbatch, format='yolo').start()
# Load detection loader
det_loader = DetectionLoader(data_loader, model=yolo_model, batchSize=args.detbatch).start()
det_processor = DetectionProcessor(det_loader).start()
runtime_profile = {
'dt': [],
'pt': [],
'pn': []
}
# Init data writer
# writer = DataWriter(args.save_video).start()
data_len = data_loader.length()
fall_res_all = []
batchSize = args.posebatch
for i in range(data_len):
start_time = getTime()
with torch.no_grad():
(inps, orig_img, im_name, boxes, scores, pt1, pt2) = det_processor.read()
if boxes is None or boxes.nelement() == 0:
# writer.save(None, None, None, None, None, orig_img, im_name.split('/')[-1])
continue
ckpt_time, det_time = getTime(start_time)
runtime_profile['dt'].append(det_time)
# Pose Estimation
# print(im_name)
datalen = inps.size(0)
leftover = 0
if (datalen) % batchSize:
leftover = 1
num_batches = datalen // batchSize + leftover
hm = []
for j in range(num_batches):
inps_j = inps[j * batchSize:min((j + 1) * batchSize, datalen)].cuda()
hm_j = pose_net(inps_j)
hm.append(hm_j)
hm = torch.cat(hm)
ckpt_time, pose_time = getTime(ckpt_time)
runtime_profile['pt'].append(pose_time)
hm = hm.cpu()
# writer.save(boxes, scores, hm, pt1, pt2, orig_img, im_name.split('/')[-1])
fall_res = []
fall_res.append(im_name.split('/')[-1])
if boxes is None:
cv2.imwrite(opt.outputpath + '/' + im_name.split('/')[-1], img)
else:
if opt.matching:
preds = getMultiPeakPrediction(
hm, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(hm, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'imgname': im_name, 'result': result}
img = vis_frame(orig_img, result)
for human in result['result']:
keypoint = human['keypoints']
keypoint = keypoint.numpy()
xmax = max(keypoint[:, 0])
xmin = min(keypoint[:, 0])
ymax = max(keypoint[:, 1])
ymin = min(keypoint[:, 1])
w = xmax - xmin
h = ymax - ymin
distance = abs((keypoint[15][1] + keypoint[16][1]) / 2 - (keypoint[11][1] + keypoint[12][1]) / 2)
if w / h >= 0.95:
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img, 'Warning!Fall', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 0, 255), 2)
fall_res.append([xmin,ymin,xmax,ymax])
'''
print('1 location:[%f,' % (xmin) + '%f]' % (ymin) + ' [%f,' % (xmax) + '%f]' % (
ymin) + ' [%f,' % (
xmin) + '%f]' % (ymax) + ' [%f,' % (xmax) + '%f]' % (ymax))
'''
else:
if distance < 55:
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img, 'Warning!Fall!', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 255, 0), 2)
fall_res.append(1)
fall_res.append([xmin,ymin,xmax,ymax])
'''
print('1 location:[%f,' % (xmin) + '%f]' % (ymin) + ' [%f,' % (xmax) + '%f]' % (
ymin) + ' [%f,' % (
xmin) + '%f]' % (ymax) + ' [%f,' % (xmax) + '%f]' % (ymax))
'''
else:
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)
#cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
print(fall_res)
cv2.imwrite(opt.outputpath + '/' + im_name.split('/')[-1], img)
ckpt_time, post_time = getTime(ckpt_time)
runtime_profile['pn'].append(post_time)
fall_res_all.append(fall_res)
return fall_res_all
def update(self):
time1 = time.time()
_, frame = self.stream.read()
# frame = cv2.resize(frame, (frame.shape[1]//2,frame.shape[0]//2))
#TODO TESTING
# frame[:,:200,:]=0
# frame[:,450:,:]=0
img_k, self.orig_img, im_dim_list_k = prep_frame(frame, self.inp_dim)
img = [img_k]
im_name = ["im_name"]
im_dim_list = [im_dim_list_k]
img = torch.cat(img)
im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2)
time2 = time.time()
with torch.no_grad():
### detector
#########################
# Human Detection
img = img.cuda()
prediction = self.det_model(img, CUDA=True)
# NMS process
dets = dynamic_write_results(prediction, opt.confidence,
opt.num_classes, nms=True, nms_conf=opt.nms_thesh)
if isinstance(dets, int) or dets.shape[0] == 0:
self.visualize2dnoperson()
return None
dets = dets.cpu()
im_dim_list = torch.index_select(im_dim_list, 0, dets[:, 0].long())
scaling_factor = torch.min(self.det_inp_dim / im_dim_list, 1)[0].view(-1, 1)
# coordinate transfer
dets[:, [1, 3]] -= (self.det_inp_dim - scaling_factor * im_dim_list[:, 0].view(-1, 1)) / 2
dets[:, [2, 4]] -= (self.det_inp_dim - scaling_factor * im_dim_list[:, 1].view(-1, 1)) / 2
dets[:, 1:5] /= scaling_factor
for j in range(dets.shape[0]):
dets[j, [1, 3]] = torch.clamp(dets[j, [1, 3]], 0.0, im_dim_list[j, 0])
dets[j, [2, 4]] = torch.clamp(dets[j, [2, 4]], 0.0, im_dim_list[j, 1])
boxes = dets[:, 1:5]
scores = dets[:, 5:6]
boxes_k = boxes[dets[:, 0] == 0]
if isinstance(boxes_k, int) or boxes_k.shape[0] == 0:
self.visualize2dnoperson()
raise NotImplementedError
return None
inps = torch.zeros(boxes_k.size(0), 3, opt.inputResH, opt.inputResW)
pt1 = torch.zeros(boxes_k.size(0), 2)
pt2 = torch.zeros(boxes_k.size(0), 2)
time3 = time.time()
### processor
#########################
inp = im_to_torch(cv2.cvtColor(self.orig_img, cv2.COLOR_BGR2RGB))
inps, pt1, pt2 = self.crop_from_dets(inp, boxes, inps, pt1, pt2)
### generator
#########################
self.orig_img = np.array(self.orig_img, dtype=np.uint8)
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
datalen = inps.size(0)
batchSize = 20 #args.posebatch()
leftover = 0
if datalen % batchSize:
leftover = 1
num_batches = datalen // batchSize + leftover
hm = []
time4 = time.time()
for j in range(num_batches):
inps_j = inps[j * batchSize:min((j + 1) * batchSize, datalen)].cuda()
hm_j = self.pose_model(inps_j)
hm.append(hm_j)
hm = torch.cat(hm)
hm = hm.cpu().data
preds_hm, preds_img, preds_scores = getPrediction(
hm, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(
boxes, scores, preds_img, preds_scores)
time5 = time.time()
if not result: # No people
self.visualize2dnoperson()
return None
else:
self.kpt = max(result,
key=lambda x: x['proposal_score'].data[0] * calculate_area(x['keypoints']), )['keypoints']
self.visualize2d()
return self.kpt
time6 = time.time()
print("process time : {} ".format(time6 - time5))
def update(self):
# keep looping infinitely
frame_prev = -1
frame_cur = 0
img_id = -1
next_id = 0
bbox_dets_list_list = []
keypoints_list_list = []
car_dets_list_list = []
car_next_id = 0
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, img_id,
CAR) = self.Q.get()
# print(img_id)
orig_img = np.array(orig_img, dtype=np.uint8)
img = orig_img
bbox_dets_list = [] # keyframe: start from empty
keypoints_list = [] # keyframe: start from empty
if boxes is None: # No person detection
bbox_det_dict = {
"img_id": img_id,
"det_id": 0,
"track_id": None,
"bbox": [0, 0, 2, 2]
}
bbox_dets_list.append(bbox_det_dict)
keypoints_dict = {
"img_id": img_id,
"det_id": 0,
"track_id": None,
"keypoints": []
}
keypoints_list.append(keypoints_dict)
bbox_dets_list_list.append(bbox_dets_list)
keypoints_list_list.append(keypoints_list)
else:
if opt.matching:
preds = getMultiPeakPrediction(hm_data, pt1.numpy(),
pt2.numpy(),
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img,
preds_scores) # list type
# 'keypoints':
# 'kp_score':
# 'proposal_score':
# 'bbox'
#
# print('boexes', boxes.size(), boxes)
# for aa in result:
# keys = aa['keypoints']
# bbox2 = aa['bbox']
# print('pose nms keys', keys.size())
# print('pose nms, box', bbox2.size(), bbox2)
#
# _result = {
# 'imgname': img_id,
# 'result': result,
# 'pt1': pt1,
# 'pt2': pt2
# }
if img_id > 0: # First frame does not have previous frame
bbox_list_prev_frame = bbox_dets_list_list[img_id -
1].copy()
keypoints_list_prev_frame = keypoints_list_list[
img_id - 1].copy()
else:
bbox_list_prev_frame = []
keypoints_list_prev_frame = []
# boxes.size(0)
num_dets = len(result)
for det_id in range(
num_dets): # detections for current frame
# obtain bbox position and track id
result_box = result[det_id]
kp_score = result_box['kp_score']
proposal_score = result_box['proposal_score'].numpy(
)[0]
if proposal_score < 1.3:
continue
keypoints = result_box['keypoints']
bbox_det = bbox_from_keypoints(keypoints) # xxyy
# enlarge bbox by 20% with same center position
# bbox_x1y1x2y2 = xywh_to_x1y1x2y2(bbox_det)
bbox_in_xywh = enlarge_bbox(bbox_det, enlarge_scale)
# print('enlared', bbox_in_xywh)
bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
# print('converted', bbox_det)
# Keyframe: use provided bbox
# if bbox_invalid(bbox_det):
# track_id = None # this id means null
# keypoints = []
# bbox_det = [0, 0, 2, 2]
# # update current frame bbox
# bbox_det_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "bbox": bbox_det}
# bbox_dets_list.append(bbox_det_dict)
# # update current frame keypoints
# keypoints_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "keypoints": keypoints}
# keypoints_list.append(keypoints_dict)
# continue
# # update current frame bbox
# obtain keypoints for each bbox position in the keyframe
# print('img id ', img_id)
if img_id == 0: # First frame, all ids are assigned automatically
track_id = next_id
next_id += 1
else:
track_id, match_index = get_track_id_SpatialConsistency(
bbox_det, bbox_list_prev_frame)
# print('track' ,track_id, match_index)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
# update current frame bbox
bbox_det_dict = {
"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"bbox": bbox_det
}
bbox_dets_list.append(bbox_det_dict)
# update current frame keypoints
keypoints_dict = {
"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"keypoints": keypoints,
'kp_score': kp_score,
'bbox': bbox_det,
'proposal_score': proposal_score
}
keypoints_list.append(keypoints_dict)
num_dets = len(bbox_dets_list)
for det_id in range(
num_dets): # detections for current frame
bbox_det_dict = bbox_dets_list[det_id]
keypoints_dict = keypoints_list[det_id]
# assert (det_id == bbox_det_dict["det_id"])
# assert (det_id == keypoints_dict["det_id"])
if bbox_det_dict[
"track_id"] == -1: # this id means matching not found yet
track_id = bbox_det_dict["track_id"]
# track_id, match_index = get_track_id_SGCN(bbox_det_dict["bbox"], bbox_list_prev_frame,
# keypoints_dict["keypoints"],
# keypoints_list_prev_frame)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
bbox_det_dict["track_id"] = track_id
keypoints_dict["track_id"] = track_id
# if still can not find a match from previous frame, then assign a new id
if track_id == -1 and not bbox_invalid(
bbox_det_dict["bbox"]):
bbox_det_dict["track_id"] = next_id
keypoints_dict["track_id"] = next_id
next_id += 1
# update frame
bbox_dets_list_list.append(bbox_dets_list)
keypoints_list_list.append(keypoints_list)
# draw keypoints
vis_frame(img, keypoints_list)
# _pt1, _pt2 = _result['pt1'].numpy(), _result['pt2'].numpy()
# pt1 = _pt1.astype(np.uint32)
# pt2 = _pt2.astype(np.uint32)
# for p1, p2 in zip(pt1, pt2):
# cv2.rectangle(img, (p1[0], p1[1]), (p2[0], p2[1]), (34, 154, 11), 1)
if CAR is not None: # No car detection
car_track_id = 0
car_np = CAR
new_car_bboxs = car_np[:, 0:4].astype(np.uint32)
new_car_score = car_np[:, 4]
car_dest_list = []
if img_id > 1: # First frame does not have previous frame
car_bbox_list_prev_frame = car_dets_list_list[
img_id - 1].copy()
else:
car_bbox_list_prev_frame = []
# print('car bbox list prev frame ', len(car_bbox_list_prev_frame))
for c, score in zip(new_car_bboxs, new_car_score):
car_bbox_det = c
bbox_in_xywh = enlarge_bbox(car_bbox_det,
enlarge_scale)
bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
# obtain keypoints for each bbox position in the keyframe
# print('img id ', img_id)
if img_id == 0: # First frame, all ids are assigned automatically
car_track_id = car_next_id
car_next_id += 1
# print('if img id zero' , car_next_id)
else:
car_track_id, match_index = get_track_id_SpatialConsistency(
bbox_det, car_bbox_list_prev_frame)
# print(car_track_id, match_index)
if car_track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del car_bbox_list_prev_frame[match_index]
bbox_det_dict = {
"img_id": img_id,
"track_id": car_track_id,
"bbox": bbox_det
}
car_dest_list.append(bbox_det_dict)
# print()
num_dets = len(car_dest_list)
for det_id in range(
num_dets): # detections for current frame
car_bbox_det_dict = car_dest_list[det_id]
# assert (det_id == bbox_det_dict["det_id"])
# assert (det_id == keypoints_dict["det_id"])
# print(Pose_matchercar_bbox_det_dict["track_id"])
if car_bbox_det_dict[
"track_id"] == -1: # this id means matching not found yet
car_bbox_det_dict["track_id"] = car_next_id
car_next_id += 1
# print('car net id ', car_next_id)
self.tracking(car_dest_list, img_id)
for car in car_dest_list:
x, y, w, h = car['bbox']
track_id = car['track_id']
tracker = self.track_dict[track_id]
history = tracker['history']
moved = np.sum(history[-10:])
last_moved = np.sum(history[-60:])
COLOR_MOVING = (0, 255, 0)
COLOR_RED = (0, 0, 255)
COLOR_INACTIVE = (255, 0, 0)
cv2.rectangle(img, (x, y), (x + w, y + h),
COLOR_INACTIVE, 1)
text_filled(img, (x, y), f'{track_id} Inactive',
COLOR_INACTIVE)
# if moved:
# cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_MOVING, 1)
# text_filled(img, (x, y), f'CAR {track_id} Active', COLOR_MOVING)
# else:
#
# if last_moved:
# cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_RED, 1)
# text_filled(img, (x, y), f'CAR {track_id} Standstill', COLOR_RED)
#
# cropped = img[y:y+h, x:x+w,:]
# filter = np.zeros(cropped.shape,dtype=img.dtype)
# # print(cropped.shape, filter.shape)
# filter[:,:,2] = 255
# # print(overlay.shape)
# # cv2.rectangle(overlay, (0, 0), (w, h), COLOR_RED, -1)
# overlayed = cv2.addWeighted(cropped,0.8,filter,0.2,0)
# img[y:y+h, x:x+w,:] = overlayed[:,:,:]
# else:
# cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_INACTIVE, 1)
# text_filled(img, (x, y), f'{track_id} Inactive', COLOR_INACTIVE)
car_dets_list_list.append(car_dest_list)
else:
car_dest_list = []
bbox_det_dict = {
"img_id": img_id,
"det_id": 0,
"track_id": None,
"bbox": [0, 0, 2, 2]
}
car_dest_list.append(bbox_det_dict)
car_dets_list_list.append(car_dest_list)
# if img_id != 0:
# for car in car_dets_list_list[-1]:
# car_track_id = car['track_id']
# if car_track_id is None:
# continue
#
# car_bbox = car['bbox']
# for human in bbox_dets_list_list[-1]:
# human_track_id = human['track_id']
# if human_track_id is None:
# continue
# hum_bbox = human['bbox']
# boxa = xywh_to_x1y1x2y2(hum_bbox)
# boxb = xywh_to_x1y1x2y2(car_bbox)
# x,y,w,h = x1y1x2y2_to_xywh(boxa)
# area = iou(boxa,boxb)
#
# if area > 0.02:
# cropped = img[y:y+h, x:x+w,:]
# filter = np.zeros(cropped.shape,dtype=img.dtype)
# filter[:,:,2] = 255
# overlayed = cv2.addWeighted(cropped,0.9,filter,0.1,0)
# img[y:y+h, x:x+w,:] = overlayed[:,:,:]
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(1)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def person_tracking(self, boxes, scores, hm_data, pt1, pt2, img_id):
person_list = []
if boxes is None:
self.person_list_list.append([])
return person_list
if opt.matching: # TODO Check the difference,
preds = getMultiPeakPrediction(hm_data, pt1.numpy(), pt2.numpy(),
opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
# result = matching(boxes, scores.numpy(), preds)
result = matching(boxes, scores, preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img,
preds_scores) # list type
# result = { 'keypoints': , 'kp_score': , 'proposal_score': , 'bbox' }
to_final_result = {'imgname': img_id, 'result': result, 'boxes': boxes}
self.final_result.append(to_final_result)
if img_id > 0: # First frame does not have previous frame
person_list_prev_frame = self.person_list_list[img_id - 1].copy()
else:
person_list_prev_frame = []
# print(result)
num_dets = len(result)
for det_id in range(
num_dets): # IOU tracking for detections in current frame.
# detections for current frame, obtain bbox position and track id
result_box = result[det_id]
kp_score = result_box['kp_score']
if opt.matching:
proposal_score = result_box['proposal_score']
else:
proposal_score = result_box['proposal_score'].numpy()[0]
if proposal_score < 0.2: # TODO check person proposal threshold
continue
if isnan(proposal_score):
continue
keypoints = result_box['keypoints'] # torch, (17,2)
keypoints_pf = np.zeros((15, 2))
idx_list = [16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9, 0, 0, 0]
for i, idx in enumerate(idx_list):
keypoints_pf[i] = keypoints[idx]
keypoints_pf[12] = (keypoints[5] + keypoints[6]) / 2 # neck
keypoints_norm = keypoints_pf - keypoints_pf[12]
# COCO-order {0-nose 1-Leye 2-Reye 3-Lear 4Rear 5-Lsho 6-Rsho 7-Lelb 8-Relb 9-Lwri 10-Rwri 11-Lhip 12-Rhip 13-Lkne 14-Rkne 15-Lank 16-Rank}
# PoseFLow order #{0-Rank 1-Rkne 2-Rhip 3-Lhip 4-Lkne 5-Lank 6-Rwri 7-Relb 8-Rsho 9-Lsho 10-Lelb 11-Lwri 12-neck 13-nose 14-TopHead}
bbox_det = bbox_from_keypoints(keypoints) # xxyy
# enlarge bbox by 20% with same center position
bbox_in_xywh = enlarge_bbox(bbox_det, enlarge_scale)
bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
# # update current frame bbox
if img_id == 0: # First frame, all ids are assigned automatically
track_id = self.person_next_id
self.person_next_id += 1
else:
track_id, match_index = get_track_id_SpatialConsistency(
bbox_det, person_list_prev_frame)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del person_list_prev_frame[match_index]
person_det_dict = {
"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"bbox": bbox_det,
"keypoints": keypoints,
'kp_norm': keypoints_norm,
'kp_poseflow': keypoints_pf,
'kp_score': kp_score,
'proposal_score': proposal_score
}
person_list.append(person_det_dict)
num_dets = len(person_list)
for det_id in range(
num_dets
): # if IOU tracking failed, run pose matching tracking.
person_dict = person_list[det_id]
if person_dict[
"track_id"] == -1: # this id means matching not found yet
# track_id = bbox_det_dict["track_id"]
track_id, match_index = get_track_id_SGCN(
person_dict["bbox"], person_list_prev_frame,
person_dict["kp_poseflow"])
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del person_list_prev_frame[match_index]
person_dict["track_id"] = track_id
else:
# if still can not find a match from previous frame, then assign a new id
# if track_id == -1 and not bbox_invalid(bbox_det_dict["bbox"]):
person_dict["track_id"] = self.person_next_id
self.person_next_id += 1
self.person_list_list.append(person_list)
return person_list
def update(self):
print(f'DataWriter_update_thread: {threading.currentThread().name}')
# keep looping infinitely
temp_kps = []
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
# 发送图像
img = orig_img
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
# 绘制床的位置矩形
# cv2.rectangle(img, (conf.Urls.bed_min_x, conf.Urls.bed_min_y),
# (conf.Urls.bed_max_x, conf.Urls.bed_max_y), (0, 255, 0), 1)
self.tcp_client.send_img(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
if len(result) > 0:
pos = result[0]['keypoints'].unsqueeze(0).numpy()
pos = self.aligner.align_points(pos)[0]
pos = (pos[..., :2] - 129) / 255
pos = torch.FloatTensor(pos)
kp = torch.cat((pos, result[0]['kp_score']), 1)
kp = kp.unsqueeze(0)
if len(temp_kps) < 9:
kp = kp.reshape([1, -1]).cuda()
temp_kps.append(kp)
kp = kp.repeat(9, 1).reshape(1, -1)
outputs = self.pos_reg_model(kp)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
result[0]['bbox'] = boxes[0]
else:
kp = kp.cuda().reshape(1, -1)
temp_kps.append(kp)
temp_kps.pop(0)
_temp_kps = torch.cat(temp_kps)
_temp_kps.cuda()
_temp_kps = _temp_kps.reshape([1, -1])
outputs = self.pos_reg_model(_temp_kps)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
result[0]['bbox'] = boxes[0]
# print(preds)
result = {'imgname': im_name, 'result': result}
self.result_Q.put(result)
self.final_result.append(result)
# 发送图像
img = vis_frame(orig_img, result)
# h, w, c = img.shape
# img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC)
# 绘制床的位置矩形
# cv2.rectangle(img, (conf.Urls.bed_min_x, conf.Urls.bed_min_y), (conf.Urls.bed_max_x, conf.Urls.bed_max_y), (0, 255, 0), 1)
self.tcp_client.send_img(img)
if opt.save_img or opt.save_video or opt.vis:
# img = vis_frame(orig_img, result)
if opt.vis:
h, w, c = img.shape
#img = cv2.resize(img, (int(w / 4), int(h / 4)), interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis',
im_name), img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
