def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
img = orig_img
cv2.waitKey(0)
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2_imshow(img)
cv2.waitKey(0)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'imgname': im_name, 'result': result}
self.final_result.append(result)
img = vis_frame(orig_img, result)
cv2_imshow(img)
cv2.waitKey(0)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
cv2_imshow(img)
cv2.waitKey(0)
if opt.vis:
cv2_imshow(img)
cv2.waitKey(0)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
print('released video stream')
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
start_time = getTime()
(boxes, scores, hm_data, pt1, pt2, orig_img, img_id, car_np) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
img = orig_img
# print('processing ' , img_id)
""" PERSON """
person_list = self.person.person_tracking(boxes, scores, hm_data, pt1, pt2, img_id)
self.person.person_tracjectory(person_list)
vis_frame(img, person_list)
""" Car """
if opt.park:
car_dest_list = self.vehicle.car_tracking(car_np, img_id)
self.vehicle.car_trajectory(car_dest_list)
self.vehicle.parking_detection(car_dest_list, img, img_id)
elif opt.fight:
self.person.fight_detection(person_list, img_id)
cv2.putText(img, f'Frame: {str(img_id).ljust(4)}', (10,40) , cv2.FONT_HERSHEY_DUPLEX, 2, WHITE, 2) # point is left-bottom
# self.fight_detection(person_list)
# self.car_person_detection(car_dest_list, bbox_dets_list, img)
ckpt_time, det_time = getTime(start_time)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(33)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def start(self):
# start to get the right result
for i in range(len(self.Q)):
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q[i]
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None or boxes.nelement() == 0:
result = {
'imgname': im_name,
'result': None,
}
self.final_result.append(result)
if opt.save_video:
self.stream.write(orig_img)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
if len(result) == 0:
result = {
'imgname': im_name,
'result': None,
}
else:
result = {
'imgname': im_name,
'result': result,
}
self.final_result.append(result)
if opt.save_video:
img = orig_img
if opt.vis and result['result'] is not None:
img = vis_frame(orig_img, result)
self.stream.write(img)
def cal_coord(pred_heatmaps, images_anno, args):
final_result = []
for img_id in pred_heatmaps.keys():
heat_h, heat_w, n_kpoints = pred_heatmaps[img_id].shape
scale_h, scale_w = heat_h / images_anno[img_id][
'height'], heat_w / images_anno[img_id]['width']
coord = []
scores = []
result = []
for p_ind in range(n_kpoints):
heat = pred_heatmaps[img_id][:, :, p_ind]
scores.append(np.max(heat))
heat = gaussian_filter(heat, sigma=5)
ind = np.unravel_index(np.argmax(heat), heat.shape)
ind = pose_processing(heat, ind)
coord_x = int((ind[1] + 1) / scale_w)
coord_y = int((ind[0] + 1) / scale_h)
coord.append((coord_x, coord_y))
result.append({'keypoints': coord, 'kp_score': scores})
result = {
'imgname': images_anno[img_id]['file_name'],
'result': result
}
if args.save_imgs:
ori_img = cv2.imread(
os.path.join(args.img_path, images_anno[img_id]['file_name']))
img = vis_frame(ori_img, result)
cv2.imshow('demo', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
cv2.imwrite(
os.path.join(args.outputpath, 'vis',
images_anno[img_id]['file_name']), img)
final_result.append(result)
return final_result
def update(self):
# keep looping infinitely
while True:
print("Q_writer:", self.Q.qsize())
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
img = orig_img
if opt.save_img or opt.save_video or opt.vis:
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'imgname': im_name, 'result': result}
self.final_result.append(result)
img, _ = vis_frame(orig_img, result)
if opt.save_img or opt.save_video or opt.vis:
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
#img = img.astype(np.uint8)
temp = cv2.imencode(".jpg", img)[1]
icode = np.array(temp)
se = icode.tostring()
n = len(se)
self.dll.mjpg_write(c_char_p(se), n)
else:
time.sleep(0.1)
def main():
# root directory
data_root = '/media/sdc1/'
# trainval is the main dataset folder for the processed videos
out_splitted = os.path.join(data_root, 'trainval')
for i, (dire, folds, fils) in enumerate(os.walk(out_splitted)):
if i == 0:
print(
'Main dataset directory: {}\n Subdirectories within it: {}\n'.
format(dire, folds))
continue
# run command on the frames
if len(folds) == 0 and len(fils) > 0:
# loop over video frames
print('Reading frames from: {}'.format(dire))
# read the annotations specific to this folder
p_json = os.path.join(dire.replace('trainval', 'trainval_anns'))
anns = load_json(p_json)
if len(anns) > 0:
# read frames
p_imgs = [
os.path.join(dire, frame_name) for frame_name in fils
if frame_name in anns
]
frames_name = [p.split('/')[-1] for p in p_imgs]
imgs = [cv2.imread(p) for p in p_imgs]
# create a list of black images
shapes = [img.shape for img in imgs]
empty = [np.zeros(shape, dtype=np.uint8) for shape in shapes]
# apply annotation
key_imgs = [
vis_frame(empty[i], anns[kd])
for i, kd in enumerate(frames_name)
]
# concatenate
img_cat = [
np.concatenate((img, key_imgs[i]), axis=1)
for i, img in enumerate(imgs)
]
p_out = os.path.join(
dire.replace('trainval', 'keypoint_frames'))
if not os.path.isdir(p_out):
os.makedirs(p_out)
# save result to disk
for i, im in enumerate(img_cat):
p = os.path.join(p_out, frames_name[i])
cv2.imwrite(p, im)
def forward(self, Q_det):
clock = 0
while 1:
with torch.no_grad():
start_time = getTime()
(inps, orig_img, boxes, scores, pt1, pt2) = Q_det.get()
ckpt_time, det_time = getTime(start_time)
img = orig_img
if boxes is not None:
hm_data = self.pose_model(inps.cuda())
ckpt_time, pose_time = getTime(ckpt_time)
hm_data = hm_data.cpu()
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'result': result}
img, datas, valid = vis_frame(img, result, opt.webcam_num)
ckpt_time, post_time = getTime(ckpt_time)
#print("det_time={:.3f}, pose_time={:.3f}, post_time={:.3f}, fps={:.1f}".format(det_time,pose_time,post_time, 1/(det_time+pose_time+post_time)))
if valid:
for data in datas:
clock = 0
self.mq.sendMsg(data)
print(data)
else:
clock = clock + 1
if clock > 500:
self.mq.sendMsg(
'{{ "source": "Behavior", "deviceId": {}, "hand": "{},{}", "body": {}, "state": {} }}'
.format(0, 0, 0, 0, 0))
clock = 0
else:
time.sleep(0.05)
clock = clock + 1
if clock > 500:
self.mq.sendMsg(
'{{ "source": "Behavior", "deviceId": {}, "hand": "{},{}", "body": {}, "state": {} }}'
.format(0, 0, 0, 0, 0))
clock = 0
''''
self.mq.sendMsg('{ "source": "Behavior", "deviceId": 0, "hand": "0,0", "body": 0 }')
print('{ "source": "Behavior", "deviceId": 0, "hand": "0,0", "body": 0 }')
'''
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(10)
temp = cv2.imencode(".jpg", img)[1]
icode = np.array(temp)
se = icode.tostring()
n = len(se)
self.dll.mjpg_write(c_char_p(se), n)
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW, opt.nClasses)
# print('dataloader:',boxes.shape, scores.shape, preds_img.shape, preds_scores.shape)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {
'imgname': im_name,
'result': result,
'boxes':boxes
}
# print('dataloader.py:result:',result)
# print('dataloader.py:hm_data.shape:',hm_data.shape)
# print('dataloader.py:preds_hm.shape:',preds_hm.shape)
# self.count+=1
# print('dataloader.py:count:',self.count)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis and len(img)!=0:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img and len(img)!=0:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video and len(img)!=0:
self.stream.write(img)
else:
time.sleep(0.1)
def update(boxes, scores, hm_data, pt1, pt2, orig_img, i):
orig_img = np.array(orig_img, dtype=np.uint8)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(
boxes, scores, preds_img, preds_scores)
result = {
'imgname': str(i),
'result': result
}
img = vis_frame(orig_img, result)
return img, result
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(result, orig_img, im_name) = self.Q.get()
if result is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
import pickle
with open("result.pkl", "wb") as f:
pickle.dump(result, f)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.05)
def cal_coord(pred_heatmaps, images_anno, args, predictions):
final_result = []
for img_id in pred_heatmaps.keys():
heat_h, heat_w, n_kpoints = pred_heatmaps[img_id].shape
scale_h, scale_w = heat_h / images_anno[img_id][
'height'], heat_w / images_anno[img_id]['width']
coord = []
scores = []
result = []
for p_ind in range(n_kpoints):
heat = pred_heatmaps[img_id][:, :, p_ind]
scores.append(np.max(heat))
heat = gaussian_filter(heat, sigma=5)
ind = np.unravel_index(np.argmax(heat), heat.shape)
ind = pose_processing(heat, ind)
coord_x = int((ind[1] + 1) / scale_w)
coord_y = int((ind[0] + 1) / scale_h)
coord.append((coord_x, coord_y))
result.append({'keypoints': coord, 'kp_score': scores})
result = {
'imgname': images_anno[img_id]['file_name'],
'result': result
}
if args.save_imgs:
if args.dataset_to_use == 'COCO':
ori_img = cv2.imread(
os.path.join(os.path.join(args.coco_dir, 'val2017'),
images_anno[img_id]['file_name']))
else:
ori_img = cv2.imread(
os.path.join(args.img_path,
images_anno[img_id]['file_name']))
img = vis_frame(ori_img, result)
img_name = images_anno[img_id]['file_name']
ind = img_name.find('.')
img_saving_name = img_name[:ind] + '{:s}'.format(
'T' if predictions[img_id] else 'F') + img_name[ind:]
cv2.imwrite(os.path.join(args.outputpath, img_saving_name), img)
final_result.append(result)
return final_result
def gen_pose(self):
with torch.no_grad():
(inps, orig_img, boxes, scores, pt1,
pt2) = self.human_detect_result
if boxes is None or boxes.nelement() == 0:
self.det_human_num = 0
self.out_img = orig_img
self.result = None
self.resultNew = None
return
#print(inps, 'inps')
#print(type(boxes), 'boxes')
#
inps = inps.cuda()
hm = self.pose_model(inps)
hm = [hm]
hm = torch.cat(hm)
hm = hm.cpu()
hm_data = hm
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
self.det_human_num = 0
self.out_img = orig_img
self.result = None
self.resultNew = None
return
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
self.result = result
self.resultNew = conserv_result_format_to_list(self.result)
resultDict = {'imgname': 'zhjs', 'result': result}
self.out_img = vis_frame(orig_img, resultDict)
self.det_human_num = len(result)
def get_pose(self, img_names):
if len(img_names) > 1:
start_lc = 4000
start_rc = 4000
now_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime())
print('========START-Ten========')
final_result = []
vis_images = []
height_difference = []
for img_index in range(len(img_names)):
print('--------------------')
img_name = img_names[img_index]
try:
img, orig_img, im_name, im_dim_list = [], [], [], []
inp_dim = int(self.args.inp_dim)
im_name_k = img_name
img_k, orig_img_k, im_dim_list_k = prep_image(
im_name_k, inp_dim)
img.append(img_k)
orig_img.append(orig_img_k)
im_name.append(im_name_k)
im_dim_list.append(im_dim_list_k)
except:
print('index-{}: image have problem'.format(img_index))
final_result.append((None, None))
continue
with torch.no_grad():
img = torch.cat(img)
im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2)
img = img.cuda()
prediction = self.det_model(img, CUDA=True)
dets = dynamic_write_results(prediction,
self.args.confidence,
self.args.num_classes,
nms=True,
nms_conf=self.args.nms_thesh)
if isinstance(dets, int) or dets.shape[0] == 0:
print('index-{}: No person detected'.format(img_index))
final_result.append((None, None))
height_difference.append(None)
continue
dets = dets.cpu()
im_dim_list = torch.index_select(im_dim_list, 0,
dets[:, 0].long())
scaling_factor = torch.min(self.det_inp_dim / im_dim_list,
1)[0].view(-1, 1)
dets[:, [1, 3]] -= (self.det_inp_dim - scaling_factor *
im_dim_list[:, 0].view(-1, 1)) / 2
dets[:, [2, 4]] -= (self.det_inp_dim - scaling_factor *
im_dim_list[:, 1].view(-1, 1)) / 2
dets[:, 1:5] /= scaling_factor
for j in range(dets.shape[0]):
dets[j, [1, 3]] = torch.clamp(dets[j, [1, 3]], 0.0,
im_dim_list[j, 0])
dets[j, [2, 4]] = torch.clamp(dets[j, [2, 4]], 0.0,
im_dim_list[j, 1])
boxes = dets[:, 1:5]
scores = dets[:, 5:6]
k = 0
boxes_k = boxes[dets[:, 0] == k]
inps = torch.zeros(boxes_k.size(0), 3, self.args.inputResH,
self.args.inputResW)
pt1 = torch.zeros(boxes_k.size(0), 2)
pt2 = torch.zeros(boxes_k.size(0), 2)
orig_img, im_name, boxes, scores, inps, pt1, pt2 = orig_img[
k], im_name[k], boxes_k, scores[dets[:, 0] ==
k], inps, pt1, pt2
inp = im_to_torch(cv2.cvtColor(orig_img,
cv2.COLOR_BGR2RGB))
inps, pt1, pt2 = crop_from_dets(inp, boxes, inps, pt1, pt2)
batchSize = self.args.posebatch
datalen = inps.size(0)
leftover = 0
if (datalen) % batchSize:
leftover = 1
num_batches = datalen // batchSize + leftover
hm = []
for j in range(num_batches):
inps_j = inps[j * batchSize:min(
(j + 1) * batchSize, datalen)].cuda()
hm_j = self.pose_model(inps_j)
hm.append(hm_j)
hm = torch.cat(hm)
hm_data = hm.cpu()
orig_img = np.array(orig_img, dtype=np.uint8)
im_name = im_name.split('/')[-1]
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, self.args.inputResH,
self.args.inputResW, self.args.outputResH,
self.args.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'imgname': im_name, 'result': result}
img = vis_frame(orig_img, result)
vis_images.append(img)
outpur_dir = os.path.join(self.args.outputpath, 'vis')
outpur_dir_raw = os.path.join(self.args.outputpath, 'raw')
if not os.path.exists(outpur_dir):
os.makedirs(outpur_dir)
if not os.path.exists(outpur_dir_raw):
os.makedirs(outpur_dir_raw)
width = img.shape[1]
keypoints = [res['keypoints'][0] for res in result['result']]
distance = [xy[0] - width / 2 for xy in keypoints]
distance = torch.tensor([torch.abs(m) for m in distance])
indice = torch.argsort(distance)[0]
pose_result = result['result'][indice]['keypoints']
# left_arm = pose_result[[6, 8, 10]].numpy()
# right_arm = pose_result[[5, 7, 9]].numpy()
# ['Nose', 'LEye', 'REye', 'LEar', 'REar', 'LShoulder', 'RShoulder', 'LElbow', 'RElbow', 'LWrist', 'RWrist', 'LHip',
# 'RHip', 'LKnee', 'RKnee', 'LAnkle', 'RAnkle']
left_arm = pose_result[[10]].numpy().astype(int)
right_arm = pose_result[[9]].numpy().astype(int)
left_arm_c_y = np.mean(left_arm, axis=0)[1]
right_arm_c_y = np.mean(right_arm, axis=0)[1]
# left_arm_c = tuple(np.mean(left_arm, axis=0).astype(int))
# right_arm_c = tuple(np.mean(right_arm, axis=0).astype(int))
left_arm_c = tuple(left_arm[0])
right_arm_c = tuple(right_arm[0])
hd = np.abs(left_arm_c_y - right_arm_c_y)
height_difference.append(hd)
cv2.circle(img, left_arm_c, 10, (0, 255, 0), -1, 8)
cv2.circle(img, right_arm_c, 10, (0, 255, 0), -1, 8)
log__vis_name = now_time + '-' + im_name
cv2.imwrite(os.path.join(outpur_dir_raw, log__vis_name),
orig_img)
cv2.imwrite(os.path.join(outpur_dir, log__vis_name), img)
if start_lc == 4000 and start_rc == 4000:
start_lc = left_arm_c_y
start_rc = right_arm_c_y
left_move = 0
right_move = 0
else:
left_move = left_arm_c_y - start_lc
right_move = right_arm_c_y - start_rc
print('index-{}--{}: left_c {:0f},right_c {:0f}'.format(
img_index, im_name, left_arm_c_y, right_arm_c_y))
print('index-{}--{}: start_lc {:0f},start_rc {:0f}'.format(
img_index, im_name, start_lc, start_rc))
print('index-{}--{}: left_move {:0f},right_move {:0f}'.format(
img_index, im_name, left_move, right_move))
print('index-{}--{}: height_difference {:0f}'.format(
img_index, im_name, hd))
final_result.append((left_move, right_move))
return final_result, vis_images, now_time, height_difference
elif len(img_names) == 1:
now_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime())
print('========START-One========')
final_result = []
vis_images = []
height_difference = []
for img_index in range(len(img_names)):
img_name = img_names[img_index]
try:
img, orig_img, im_name, im_dim_list = [], [], [], []
inp_dim = int(self.args.inp_dim)
im_name_k = img_name
img_k, orig_img_k, im_dim_list_k = prep_image(
im_name_k, inp_dim)
img.append(img_k)
orig_img.append(orig_img_k)
im_name.append(im_name_k)
im_dim_list.append(im_dim_list_k)
except:
print('index-{}: image have problem'.format(img_index))
final_result.append((None, None))
with torch.no_grad():
img = torch.cat(img)
vis_img = img.numpy()[0]
vis_img = np.transpose(vis_img, (1, 2, 0))
vis_img = vis_img[:, :, ::-1]
vis_images.append(vis_img)
im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2)
img = img.cuda()
prediction = self.det_model(img, CUDA=True)
dets = dynamic_write_results(prediction,
self.args.confidence,
self.args.num_classes,
nms=True,
nms_conf=self.args.nms_thesh)
if isinstance(dets, int) or dets.shape[0] == 0:
print('index-{}: No person detected'.format(img_index))
final_result.append((None, None))
else:
print('index-{}: Person detected'.format(img_index))
final_result.append((4, 4))
return final_result, vis_images, now_time, height_difference
def update(self):
next_id = 0
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
start_time = getTime()
(boxes, scores, hm_data, pt1, pt2, orig_img, img_id,
CAR) = self.Q.get()
# print(img_id)
orig_img = np.array(orig_img, dtype=np.uint8)
img = orig_img
# text_filled2(img,(5,200),str(img_id),LIGHT_GREEN,2,2)
"""
PERSON
"""
if boxes is None: # No person detection
person_list = []
pass
else:
person_list = self.person_tracking(boxes, scores, hm_data,
pt1, pt2, img_id)
# update frame
vis_frame(img, person_list)
"""
Car
"""
if CAR is not None:
car_np = CAR
car_dest_list = self.car_tracking(car_np, img_id)
else:
car_dest_list = []
self.car_list_list.append(car_dest_list)
self.person_list_list.append(person_list)
self.car_trajectory(car_dest_list)
self.person_tracjectory(person_list)
# FOR GIST2019
if img_id != 0:
self.fight_detection(person_list)
# self.parking_detection(car_dest_list, img, img_id)
# self.car_person_detection(car_dest_list, bbox_dets_list, img)
# FOR NEXPA
# self.person_nexpa(bbox_dets_list,img,img_id)
ckpt_time, det_time = getTime(start_time)
# cv2.putText(img, str(1 / det_time), (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1)
cv2.putText(img, str(img_id), (5, 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(33)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
count = 0
# filepath = '/home/yurik/Documents/Program/Alphapose_zed_video/testdata/20191014/walkstraightly/walkstraightly.svo'
# init = sl.InitParameters(svo_input_filename=filepath,svo_real_time_mode=False)
# init.depth_mode = sl.DEPTH_MODE.DEPTH_MODE_QUALITY
# cam = sl.Camera()
# runtime = sl.RuntimeParameters()
# status = cam.open(init)
# mat = sl.Mat()
# zeroarr = np.zeros((720,1280,3))
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
if opt.matching:
preds = getMultiPeakPrediction(
hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(
boxes, scores, preds_img, preds_scores)
result = {
'imgname': im_name,
'result': result
}
# 3D coordinates computation
ppl = result['result']
ppl_num = len(ppl)
self.coordinates_u, self.coordinates_v, self.truex, self.truey, self.dists = fl.people_3d_coord(ppl, ppl_num,
self.video_mode, self.camMtx1, orig_img)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
# err = cam.grab(runtime)
# if err == sl.ERROR_CODE.SUCCESS:
# cam.retrieve_image(mat, sl.VIEW.VIEW_DEPTH)
# depthmap = mat.get_data()
# if img.shape[2] == 3:
# depthmap = cv2.cvtColor(depthmap, cv2.COLOR_RGBA2RGB)
# depthmap = cv2.resize(depthmap, (int(img.shape[1]/2), img.shape[0]))
# depthmap = cv2.applyColorMap(depthmap, cv2.COLORMAP_JET)
# depthmap = np.hstack((depthmap, zeroarr))
# depthmap = depthmap.astype(np.uint8)
# img = cv2.addWeighted(img, 0.5, depthmap, 0.5, 3)
if len(self.coordinates_v) > 0 and len(self.coordinates_u) > 0:
for i in range(len(self.coordinates_v)):
# cv2.putText(img, 'z:' + str(round((self.dists[i] / 10), 1)),
# (int(self.coordinates_u[i]), int(self.coordinates_v[i]) - 15),
# cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 3, 8)
cv2.putText(img, str(round((self.truex[i] / 10), 1)),
(int(self.coordinates_u[i]), int(self.coordinates_v[i]) - 15),
cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 3, 8)
cv2.putText(img, str(round((self.truey[i] / 10), 1)),
(int(self.coordinates_u[i]) + 200, int(self.coordinates_v[i]) - 15),
cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 3, 8)
cv2.putText(img, str(round((self.dists[i] / 10), 1)),
(int(self.coordinates_u[i]) + 400, int(self.coordinates_v[i]) - 15),
cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 3, 8)
cv2.putText(img, 'frames: ' + str(count), (620, 620), cv2.FONT_HERSHEY_PLAIN, 2, (0,100,90), 3, 8)
else:
cv2.putText(img, '[N/A]',
(40, 620), cv2.FONT_HERSHEY_PLAIN, 2, (0, 100, 90), 3, 8)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
count = count + 1
keypoints = d[i]['keypoints']
dim = len(keypoints)
kp_scores = []
for i in range(len(keypoints) - 1, 0, -3):
kp_scores.append(keypoints.pop(i))
kp_scores.reverse()
kp_scores = torch.FloatTensor(kp_scores)
kp_preds = np.reshape(keypoints, (int(dim / 3), 2))
# kp_preds = list(kp_preds)
kp_preds = torch.FloatTensor(kp_preds)
human = {'keypoints': kp_preds, 'kp_score': kp_scores}
result.append(human)
all_result = {'imgname': d[0]['image_id'], 'result': result}
# print(all_result['result'])
img = fn.vis_frame(image, all_result)
cv2.imshow('img', img)
cv2.waitKey(0)
print('//////////////////////////////////')
# kp_scores = []
# for i in range(len(keypoints)-1, 0, -3):
# kp_scores.append( keypoints.pop(i) )
# kp_scores.reverse()
# kp_preds = np.reshape(keypoints,(17,2))
# human = {'keypoints':kp_preds, 'kp_score':kp_scores}
print('///////////////////////')
# print(human['keypoints'])
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
'''
with open("IP2.csv",'a',newline='') as t:
writer=csv.writer(t)
writer.writerow([0])
'''
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(0)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
if opt.matching:
preds = getMultiPeakPrediction(hm_data, pt1.numpy(),
pt2.numpy(),
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img,
preds_scores)
result = {'imgname': im_name, 'result': result}
'''
with open("IP2.csv",'a',newline='') as t:
writer=csv.writer(t)
writer.writerow([data])
'''
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img, data = vis_frame(orig_img, result)
#img = vis_frame(orig_img, result) #draw human pose limbs and keypoints
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
temp_kps = []
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
#print(len(result))
#print(boxes.shape)
result = []
if preds_img.shape[0] > 0:
for re in range(preds_img.shape[0]):
pos = preds_img[re].unsqueeze(0).numpy()
pos = self.aligner.align_points(pos)[0]
pos = (pos[..., :2] - 129) / 255
pos = torch.FloatTensor(pos)
kp = torch.cat(
(pos, preds_scores[re].unsqueeze(1)), 1)
kp = kp.unsqueeze(0)
kp = kp.reshape([1, -1]).cuda()
kp = kp.repeat(9, 1).reshape(1, -1)
outputs = self.pos_reg_model(kp)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result.append({
'class':
str(float(classidx)),
'keypoints':
preds_img[re],
'kp_score':
preds_scores[re].unsqueeze(1),
'bbox':
boxes[re]
})
# print(preds)
result = {'imgname': im_name, 'result': result}
# if len(result)>0:
# for re in range(len(result)):
# pos = result[re]['keypoints'].unsqueeze(0).numpy()
# pos = self.aligner.align_points(pos)[0]
# pos = (pos[..., :2] - 129) / 255
# pos = torch.FloatTensor(pos)
# kp = torch.cat((pos, result[0]['kp_score']), 1)
# kp = kp.unsqueeze(0)
# kp = kp.reshape([1, -1]).cuda()
# kp = kp.repeat(9, 1).reshape(1, -1)
# outputs = self.pos_reg_model(kp)
# _, preds = torch.max(outputs, 1)
# classidx = preds.cpu()
# result[re]['class'] = str(float(classidx))
# # print(preds)
# result = {
# 'imgname': im_name,
# 'result': result
# }
self.result_Q.put(result)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis',
im_name), img)
if opt.save_video:
self.stream.write(img)
# 发送图像
img = vis_frame(orig_img, result)
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
self.tcp_client.send_img(img)
else:
time.sleep(0.1)
size = ((int(device.get(cv2.CAP_PROP_FRAME_WIDTH)),int(device.get(cv2.CAP_PROP_FRAME_HEIGHT))))
success, frame = device.read()
###################################################################################
success = True
frame_id = 0
while success:
success, frame = device.read()
img_final_list = find_athlete(test[frame_id], my_matrix, tennis_width, tennis_height)
# frame_queue.put(frame)
frame_id = frame_id + 1
img_final_list = find_athlete(test[150], my_matrix, tennis_width, tennis_height)
img = fn.vis_frame(image,test[150])
# cv2.imshow('img', img)
# cv2.waitKey(0)
###########################################################
# resize target image. Don't care this part:
court = cv2.imread("court|.png")
# court = court[140:705, 160:390, :] # range of height and width
court = court[140:705, 164:387, :]
scale = (tennis_width, tennis_height)
court = cv2.resize(court, scale)
###########################################################
# cv2.circle(court, people_transf_position, 3, (0,0,0),10)
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {
'imgname': im_name,
'result': result
} # append imgname here.
# result here includes imgname, bbox, kps, kp_score, proposal_score
# Critical, run pnp algorithm here to get 6d pose.
# embed()
if result['result']:
kp_2d = np.array(result['result'][0]['keypoints'])
kp_3d = np.array(self.kp_3d)
R, t = pnp(kp_3d, kp_2d, self.cam_K)
result.update({'cam_R': R, 'cam_t': t})
else:
result.update({'cam_R': [], 'cam_t': []})
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
next_id = 0
car_next_id = 0
bbox_dets_list_list = []
keypoints_list_list = []
car_dets_list_list = []
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
start_time = getTime()
(boxes, scores, hm_data, pt1, pt2, orig_img, img_id, CAR) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is not None:
boxes = boxes.astype(np.int32)
img = orig_img
# text_filled2(img,(5,200),str(img_id),LIGHT_GREEN,2,2)
bbox_dets_list = [] # keyframe: start from empty
keypoints_list = [] # keyframe: start from empty
# print(boxes)
if boxes is None: # No person detection
pass
# bbox_det_dict = {"img_id": img_id,
# "det_id": 0,
# "track_id": None,
# "bbox": [0, 0, 2, 2]}
# bbox_dets_list.append(bbox_det_dict)
#
# keypoints_dict = {"img_id": img_id,
# "det_id": 0,
# "track_id": None,
# "keypoints": []}
# keypoints_list.append(keypoints_dict)
else:
if opt.matching:
preds = getMultiPeakPrediction(
hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(hm_data, pt1, pt2, opt.inputResH,
opt.inputResW, opt.outputResH,
opt.outputResW)
# print('number of result', preds_hm, preds_scores )
result = pose_nms(boxes, scores, preds_img, preds_scores) # list type
# result = { 'keypoints': , 'kp_score': , 'proposal_score': , 'bbox' }
if img_id > 0: # First frame does not have previous frame
bbox_list_prev_frame = bbox_dets_list_list[img_id - 1].copy()
keypoints_list_prev_frame = keypoints_list_list[img_id - 1].copy()
else:
bbox_list_prev_frame = []
keypoints_list_prev_frame = []
# boxes.size(0)
num_dets = len(result)
for bbox in boxes:
x, y, w, h = bbox.astype(np.uint32)
cv2.rectangle(orig_img, (x, y), (x + w, y + h), (253, 222, 111), 1)
for det_id in range(num_dets): # IOU tracking for detections in current frame.
# detections for current frame
# obtain bbox position and track id
result_box = result[det_id]
kp_score = result_box['kp_score']
proposal_score = result_box['proposal_score'].numpy()[0]
if proposal_score < 1.3:
continue
keypoints = result_box['keypoints'] # torch, (17,2)
keypoints_pf = np.zeros((15, 2))
idx_list = [16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9, 0, 0, 0]
for i, idx in enumerate(idx_list):
keypoints_pf[i] = keypoints[idx]
keypoints_pf[12] = (keypoints[5] + keypoints[6]) / 2 # neck
# COCO-order {0-nose 1-Leye 2-Reye 3-Lear 4Rear 5-Lsho 6-Rsho 7-Lelb 8-Relb 9-Lwri 10-Rwri 11-Lhip 12-Rhip 13-Lkne 14-Rkne 15-Lank 16-Rank}
# PoseFLow order #{0-Rank 1-Rkne 2-Rhip 3-Lhip 4-Lkne 5-Lank 6-Rwri 7-Relb 8-Rsho 9-Lsho 10-Lelb 11-Lwri 12-neck 13-nose 14-TopHead}
bbox_det = bbox_from_keypoints(keypoints) # xxyy
# bbox_in_xywh = enlarge_bbox(bbox_det, enlarge_scale)
# bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
# Keyframe: use provided bbox
# if bbox_invalid(bbox_det):
# track_id = None # this id means null
# keypoints = []
# bbox_det = [0, 0, 2, 2]
# # update current frame bbox
# bbox_det_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "bbox": bbox_det}
# bbox_dets_list.append(bbox_det_dict)
# # update current frame keypoints
# keypoints_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "keypoints": keypoints}
# keypoints_list.append(keypoints_dict)
# continue
# # update current frame bbox
if img_id == 0: # First frame, all ids are assigned automatically
track_id = next_id
next_id += 1
else:
track_id, match_index = get_track_id_SpatialConsistency(bbox_det, bbox_list_prev_frame)
# print('track' ,track_id, match_index)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
# update current frame bbox
bbox_det_dict = {"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"bbox": bbox_det}
# update current frame keypoints
keypoints_dict = {"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"keypoints": keypoints,
'kp_poseflow': keypoints_pf,
'kp_score': kp_score,
'bbox': bbox_det,
'proposal_score': proposal_score}
bbox_dets_list.append(bbox_det_dict)
keypoints_list.append(keypoints_dict)
num_dets = len(bbox_dets_list)
for det_id in range(num_dets): # if IOU tracking failed, run pose matching tracking.
bbox_det_dict = bbox_dets_list[det_id]
keypoints_dict = keypoints_list[det_id]
# assert (det_id == bbox_det_dict["det_id"])
# assert (det_id == keypoints_dict["det_id"])
if bbox_det_dict["track_id"] == -1: # this id means matching not found yet
# track_id = bbox_det_dict["track_id"]
track_id, match_index = get_track_id_SGCN(bbox_det_dict["bbox"], bbox_list_prev_frame,
keypoints_dict["kp_poseflow"],
keypoints_list_prev_frame)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
bbox_det_dict["track_id"] = track_id
keypoints_dict["track_id"] = track_id
# if still can not find a match from previous frame, then assign a new id
# if track_id == -1 and not bbox_invalid(bbox_det_dict["bbox"]):
if track_id == -1:
bbox_det_dict["track_id"] = next_id
keypoints_dict["track_id"] = next_id
next_id += 1
# update frame
# print('keypoint list', len(keypoints_list))
vis_frame(img, keypoints_list)
"""
Car
"""
if CAR is not None:
car_np = CAR
new_car_bboxs = car_np[:, 0:4].astype(np.uint32) # b/ x y w h c / cls_conf, cls_idx
new_car_score = car_np[:, 4]
cls_conf = car_np[:, 4]
# print("id: ", img_id , " ------------ " , new_car_bboxs, new_car_score)
# cls_conf = car_np[:, 6]
car_dest_list = []
if img_id > 1: # First frame does not have previous frame
car_bbox_list_prev_frame = car_dets_list_list[img_id - 1].copy()
else:
car_bbox_list_prev_frame = []
# print('car bbox list prev frame ', len(car_bbox_list_prev_frame))
for c, score, conf in zip(new_car_bboxs, new_car_score, cls_conf):
# car_bbox_det = c
# car_bbox_det = x1y1x2y2_to_xywh(c)
bbox_det = c
# bbox_in_xywh = enlarge_bbox(car_bbox_det, enlarge_scale)
# bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
if img_id == 0: # First frame, all ids are assigned automatically
car_track_id = car_next_id
car_next_id += 1
else:
car_track_id, match_index = get_track_id_SpatialConsistency(bbox_det,
car_bbox_list_prev_frame)
# print(car_track_id, match_index)
if car_track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del car_bbox_list_prev_frame[match_index]
bbox_det_dict = {"img_id": img_id,
"track_id": car_track_id,
"bbox": bbox_det,
"score": score,
"conf": conf}
car_dest_list.append(bbox_det_dict)
for car_bbox_det_dict in car_dest_list: # detections for current frame
if car_bbox_det_dict["track_id"] == -1: # this id means matching not found yet
car_bbox_det_dict["track_id"] = car_next_id
car_next_id += 1
self.tracking(car_dest_list)
car_dets_list_list.append(car_dest_list)
else:
car_dest_list = []
bbox_det_dict = {"img_id": img_id,
"det_id": 0,
"track_id": None,
"bbox": [0, 0, 2, 2],
"score": 0,
"conf": 0}
car_dest_list.append(bbox_det_dict)
car_dets_list_list.append(car_dest_list)
bbox_dets_list_list.append(bbox_dets_list)
keypoints_list_list.append(keypoints_list)
if img_id != 0:
self.car_person_detection(car_dest_list, bbox_dets_list, img)
self.car_parking_detection(car_dest_list, img, img_id)
ckpt_time, det_time = getTime(start_time)
cv2.putText(img, str(1 / det_time), (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(33)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
print(f'DataWriter_update_thread: {threading.currentThread().name}')
# keep looping infinitely
temp_kps = []
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
# 发送图像
img = orig_img
h, w, c = img.shape
img = cv2.resize(img, (int(w / 2), int(h / 2)),
interpolation=cv2.INTER_CUBIC)
# 绘制床的位置矩形
# cv2.rectangle(img, (conf.Urls.bed_min_x, conf.Urls.bed_min_y),
# (conf.Urls.bed_max_x, conf.Urls.bed_max_y), (0, 255, 0), 1)
self.tcp_client.send_img(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
if len(result) > 0:
pos = result[0]['keypoints'].unsqueeze(0).numpy()
pos = self.aligner.align_points(pos)[0]
pos = (pos[..., :2] - 129) / 255
pos = torch.FloatTensor(pos)
kp = torch.cat((pos, result[0]['kp_score']), 1)
kp = kp.unsqueeze(0)
if len(temp_kps) < 9:
kp = kp.reshape([1, -1]).cuda()
temp_kps.append(kp)
kp = kp.repeat(9, 1).reshape(1, -1)
outputs = self.pos_reg_model(kp)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
result[0]['bbox'] = boxes[0]
else:
kp = kp.cuda().reshape(1, -1)
temp_kps.append(kp)
temp_kps.pop(0)
_temp_kps = torch.cat(temp_kps)
_temp_kps.cuda()
_temp_kps = _temp_kps.reshape([1, -1])
outputs = self.pos_reg_model(_temp_kps)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
result[0]['bbox'] = boxes[0]
# print(preds)
result = {'imgname': im_name, 'result': result}
self.result_Q.put(result)
self.final_result.append(result)
# 发送图像
img = vis_frame(orig_img, result)
# h, w, c = img.shape
# img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC)
# 绘制床的位置矩形
# cv2.rectangle(img, (conf.Urls.bed_min_x, conf.Urls.bed_min_y), (conf.Urls.bed_max_x, conf.Urls.bed_max_y), (0, 255, 0), 1)
self.tcp_client.send_img(img)
if opt.save_img or opt.save_video or opt.vis:
# img = vis_frame(orig_img, result)
if opt.vis:
h, w, c = img.shape
#img = cv2.resize(img, (int(w / 4), int(h / 4)), interpolation=cv2.INTER_CUBIC)
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis',
im_name), img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
frame_prev = -1
frame_cur = 0
img_id = -1
next_id = 0
bbox_dets_list_list = []
keypoints_list_list = []
car_dets_list_list = []
car_next_id = 0
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, img_id,
CAR) = self.Q.get()
# print(img_id)
orig_img = np.array(orig_img, dtype=np.uint8)
img = orig_img
bbox_dets_list = [] # keyframe: start from empty
keypoints_list = [] # keyframe: start from empty
if boxes is None: # No person detection
bbox_det_dict = {
"img_id": img_id,
"det_id": 0,
"track_id": None,
"bbox": [0, 0, 2, 2]
}
bbox_dets_list.append(bbox_det_dict)
keypoints_dict = {
"img_id": img_id,
"det_id": 0,
"track_id": None,
"keypoints": []
}
keypoints_list.append(keypoints_dict)
bbox_dets_list_list.append(bbox_dets_list)
keypoints_list_list.append(keypoints_list)
else:
if opt.matching:
preds = getMultiPeakPrediction(hm_data, pt1.numpy(),
pt2.numpy(),
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img,
preds_scores) # list type
# 'keypoints':
# 'kp_score':
# 'proposal_score':
# 'bbox'
#
# print('boexes', boxes.size(), boxes)
# for aa in result:
# keys = aa['keypoints']
# bbox2 = aa['bbox']
# print('pose nms keys', keys.size())
# print('pose nms, box', bbox2.size(), bbox2)
#
# _result = {
# 'imgname': img_id,
# 'result': result,
# 'pt1': pt1,
# 'pt2': pt2
# }
if img_id > 0: # First frame does not have previous frame
bbox_list_prev_frame = bbox_dets_list_list[img_id -
1].copy()
keypoints_list_prev_frame = keypoints_list_list[
img_id - 1].copy()
else:
bbox_list_prev_frame = []
keypoints_list_prev_frame = []
# boxes.size(0)
num_dets = len(result)
for det_id in range(
num_dets): # detections for current frame
# obtain bbox position and track id
result_box = result[det_id]
kp_score = result_box['kp_score']
proposal_score = result_box['proposal_score'].numpy(
)[0]
if proposal_score < 1.3:
continue
keypoints = result_box['keypoints']
bbox_det = bbox_from_keypoints(keypoints) # xxyy
# enlarge bbox by 20% with same center position
# bbox_x1y1x2y2 = xywh_to_x1y1x2y2(bbox_det)
bbox_in_xywh = enlarge_bbox(bbox_det, enlarge_scale)
# print('enlared', bbox_in_xywh)
bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
# print('converted', bbox_det)
# Keyframe: use provided bbox
# if bbox_invalid(bbox_det):
# track_id = None # this id means null
# keypoints = []
# bbox_det = [0, 0, 2, 2]
# # update current frame bbox
# bbox_det_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "bbox": bbox_det}
# bbox_dets_list.append(bbox_det_dict)
# # update current frame keypoints
# keypoints_dict = {"img_id": img_id,
# "det_id": det_id,
# "track_id": track_id,
# "keypoints": keypoints}
# keypoints_list.append(keypoints_dict)
# continue
# # update current frame bbox
# obtain keypoints for each bbox position in the keyframe
# print('img id ', img_id)
if img_id == 0: # First frame, all ids are assigned automatically
track_id = next_id
next_id += 1
else:
track_id, match_index = get_track_id_SpatialConsistency(
bbox_det, bbox_list_prev_frame)
# print('track' ,track_id, match_index)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
# update current frame bbox
bbox_det_dict = {
"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"bbox": bbox_det
}
bbox_dets_list.append(bbox_det_dict)
# update current frame keypoints
keypoints_dict = {
"img_id": img_id,
"det_id": det_id,
"track_id": track_id,
"keypoints": keypoints,
'kp_score': kp_score,
'bbox': bbox_det,
'proposal_score': proposal_score
}
keypoints_list.append(keypoints_dict)
num_dets = len(bbox_dets_list)
for det_id in range(
num_dets): # detections for current frame
bbox_det_dict = bbox_dets_list[det_id]
keypoints_dict = keypoints_list[det_id]
# assert (det_id == bbox_det_dict["det_id"])
# assert (det_id == keypoints_dict["det_id"])
if bbox_det_dict[
"track_id"] == -1: # this id means matching not found yet
track_id = bbox_det_dict["track_id"]
# track_id, match_index = get_track_id_SGCN(bbox_det_dict["bbox"], bbox_list_prev_frame,
# keypoints_dict["keypoints"],
# keypoints_list_prev_frame)
if track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del bbox_list_prev_frame[match_index]
del keypoints_list_prev_frame[match_index]
bbox_det_dict["track_id"] = track_id
keypoints_dict["track_id"] = track_id
# if still can not find a match from previous frame, then assign a new id
if track_id == -1 and not bbox_invalid(
bbox_det_dict["bbox"]):
bbox_det_dict["track_id"] = next_id
keypoints_dict["track_id"] = next_id
next_id += 1
# update frame
bbox_dets_list_list.append(bbox_dets_list)
keypoints_list_list.append(keypoints_list)
# draw keypoints
vis_frame(img, keypoints_list)
# _pt1, _pt2 = _result['pt1'].numpy(), _result['pt2'].numpy()
# pt1 = _pt1.astype(np.uint32)
# pt2 = _pt2.astype(np.uint32)
# for p1, p2 in zip(pt1, pt2):
# cv2.rectangle(img, (p1[0], p1[1]), (p2[0], p2[1]), (34, 154, 11), 1)
if CAR is not None: # No car detection
car_track_id = 0
car_np = CAR
new_car_bboxs = car_np[:, 0:4].astype(np.uint32)
new_car_score = car_np[:, 4]
car_dest_list = []
if img_id > 1: # First frame does not have previous frame
car_bbox_list_prev_frame = car_dets_list_list[
img_id - 1].copy()
else:
car_bbox_list_prev_frame = []
# print('car bbox list prev frame ', len(car_bbox_list_prev_frame))
for c, score in zip(new_car_bboxs, new_car_score):
car_bbox_det = c
bbox_in_xywh = enlarge_bbox(car_bbox_det,
enlarge_scale)
bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh)
# obtain keypoints for each bbox position in the keyframe
# print('img id ', img_id)
if img_id == 0: # First frame, all ids are assigned automatically
car_track_id = car_next_id
car_next_id += 1
# print('if img id zero' , car_next_id)
else:
car_track_id, match_index = get_track_id_SpatialConsistency(
bbox_det, car_bbox_list_prev_frame)
# print(car_track_id, match_index)
if car_track_id != -1: # if candidate from prev frame matched, prevent it from matching another
del car_bbox_list_prev_frame[match_index]
bbox_det_dict = {
"img_id": img_id,
"track_id": car_track_id,
"bbox": bbox_det
}
car_dest_list.append(bbox_det_dict)
# print()
num_dets = len(car_dest_list)
for det_id in range(
num_dets): # detections for current frame
car_bbox_det_dict = car_dest_list[det_id]
# assert (det_id == bbox_det_dict["det_id"])
# assert (det_id == keypoints_dict["det_id"])
# print(Pose_matchercar_bbox_det_dict["track_id"])
if car_bbox_det_dict[
"track_id"] == -1: # this id means matching not found yet
car_bbox_det_dict["track_id"] = car_next_id
car_next_id += 1
# print('car net id ', car_next_id)
self.tracking(car_dest_list, img_id)
for car in car_dest_list:
x, y, w, h = car['bbox']
track_id = car['track_id']
tracker = self.track_dict[track_id]
history = tracker['history']
moved = np.sum(history[-10:])
last_moved = np.sum(history[-60:])
COLOR_MOVING = (0, 255, 0)
COLOR_RED = (0, 0, 255)
COLOR_INACTIVE = (255, 0, 0)
cv2.rectangle(img, (x, y), (x + w, y + h),
COLOR_INACTIVE, 1)
text_filled(img, (x, y), f'{track_id} Inactive',
COLOR_INACTIVE)
# if moved:
# cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_MOVING, 1)
# text_filled(img, (x, y), f'CAR {track_id} Active', COLOR_MOVING)
# else:
#
# if last_moved:
# cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_RED, 1)
# text_filled(img, (x, y), f'CAR {track_id} Standstill', COLOR_RED)
#
# cropped = img[y:y+h, x:x+w,:]
# filter = np.zeros(cropped.shape,dtype=img.dtype)
# # print(cropped.shape, filter.shape)
# filter[:,:,2] = 255
# # print(overlay.shape)
# # cv2.rectangle(overlay, (0, 0), (w, h), COLOR_RED, -1)
# overlayed = cv2.addWeighted(cropped,0.8,filter,0.2,0)
# img[y:y+h, x:x+w,:] = overlayed[:,:,:]
# else:
# cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_INACTIVE, 1)
# text_filled(img, (x, y), f'{track_id} Inactive', COLOR_INACTIVE)
car_dets_list_list.append(car_dest_list)
else:
car_dest_list = []
bbox_det_dict = {
"img_id": img_id,
"det_id": 0,
"track_id": None,
"bbox": [0, 0, 2, 2]
}
car_dest_list.append(bbox_det_dict)
car_dets_list_list.append(car_dest_list)
# if img_id != 0:
# for car in car_dets_list_list[-1]:
# car_track_id = car['track_id']
# if car_track_id is None:
# continue
#
# car_bbox = car['bbox']
# for human in bbox_dets_list_list[-1]:
# human_track_id = human['track_id']
# if human_track_id is None:
# continue
# hum_bbox = human['bbox']
# boxa = xywh_to_x1y1x2y2(hum_bbox)
# boxb = xywh_to_x1y1x2y2(car_bbox)
# x,y,w,h = x1y1x2y2_to_xywh(boxa)
# area = iou(boxa,boxb)
#
# if area > 0.02:
# cropped = img[y:y+h, x:x+w,:]
# filter = np.zeros(cropped.shape,dtype=img.dtype)
# filter[:,:,2] = 255
# overlayed = cv2.addWeighted(cropped,0.9,filter,0.1,0)
# img[y:y+h, x:x+w,:] = overlayed[:,:,:]
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(1)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def detect_main(args, im_names, yolo_model, pose_net):
# Load input images
data_loader = ImageLoader(im_names, batchSize=args.detbatch, format='yolo').start()
# Load detection loader
det_loader = DetectionLoader(data_loader, model=yolo_model, batchSize=args.detbatch).start()
det_processor = DetectionProcessor(det_loader).start()
runtime_profile = {
'dt': [],
'pt': [],
'pn': []
}
# Init data writer
# writer = DataWriter(args.save_video).start()
data_len = data_loader.length()
fall_res_all = []
batchSize = args.posebatch
for i in range(data_len):
start_time = getTime()
with torch.no_grad():
(inps, orig_img, im_name, boxes, scores, pt1, pt2) = det_processor.read()
if boxes is None or boxes.nelement() == 0:
# writer.save(None, None, None, None, None, orig_img, im_name.split('/')[-1])
continue
ckpt_time, det_time = getTime(start_time)
runtime_profile['dt'].append(det_time)
# Pose Estimation
# print(im_name)
datalen = inps.size(0)
leftover = 0
if (datalen) % batchSize:
leftover = 1
num_batches = datalen // batchSize + leftover
hm = []
for j in range(num_batches):
inps_j = inps[j * batchSize:min((j + 1) * batchSize, datalen)].cuda()
hm_j = pose_net(inps_j)
hm.append(hm_j)
hm = torch.cat(hm)
ckpt_time, pose_time = getTime(ckpt_time)
runtime_profile['pt'].append(pose_time)
hm = hm.cpu()
# writer.save(boxes, scores, hm, pt1, pt2, orig_img, im_name.split('/')[-1])
fall_res = []
fall_res.append(im_name.split('/')[-1])
if boxes is None:
cv2.imwrite(opt.outputpath + '/' + im_name.split('/')[-1], img)
else:
if opt.matching:
preds = getMultiPeakPrediction(
hm, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(hm, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'imgname': im_name, 'result': result}
img = vis_frame(orig_img, result)
for human in result['result']:
keypoint = human['keypoints']
keypoint = keypoint.numpy()
xmax = max(keypoint[:, 0])
xmin = min(keypoint[:, 0])
ymax = max(keypoint[:, 1])
ymin = min(keypoint[:, 1])
w = xmax - xmin
h = ymax - ymin
distance = abs((keypoint[15][1] + keypoint[16][1]) / 2 - (keypoint[11][1] + keypoint[12][1]) / 2)
if w / h >= 0.95:
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img, 'Warning!Fall', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 0, 255), 2)
fall_res.append([xmin,ymin,xmax,ymax])
'''
print('1 location:[%f,' % (xmin) + '%f]' % (ymin) + ' [%f,' % (xmax) + '%f]' % (
ymin) + ' [%f,' % (
xmin) + '%f]' % (ymax) + ' [%f,' % (xmax) + '%f]' % (ymax))
'''
else:
if distance < 55:
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img, 'Warning!Fall!', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 255, 0), 2)
fall_res.append(1)
fall_res.append([xmin,ymin,xmax,ymax])
'''
print('1 location:[%f,' % (xmin) + '%f]' % (ymin) + ' [%f,' % (xmax) + '%f]' % (
ymin) + ' [%f,' % (
xmin) + '%f]' % (ymax) + ' [%f,' % (xmax) + '%f]' % (ymax))
'''
else:
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)
#cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
print(fall_res)
cv2.imwrite(opt.outputpath + '/' + im_name.split('/')[-1], img)
ckpt_time, post_time = getTime(ckpt_time)
runtime_profile['pn'].append(post_time)
fall_res_all.append(fall_res)
return fall_res_all
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
# print('++++++++++++++++++++we will print original image+++++++++++++++++++++++++')
# cv2.imwrite(os.path.join(opt.outputpath, 'det_img', im_name), orig_img)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
if opt.matching:
preds = getMultiPeakPrediction(hm_data, pt1.numpy(),
pt2.numpy(),
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
pose_result = pose_nms(boxes, scores, preds_img,
preds_scores)
result = {'imgname': im_name, 'result': pose_result}
# occlusion evalution
result = occlud_eval(result)
'''pan edit on 20200406'''
# bbox semantic code
result = bbox_code_process(result)
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(opt.outputpath, 'vis', im_name),
img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
coords = []
result = []
for i in range(num_kps):
heat = heatmap[:, :, i]
heat = gaussian_filter(heat, sigma=5)
ind = pose_processing(heat, max_locs[i])
coord_x = int((ind[1] + 1) / scale_w)
coord_y = int((ind[0] + 1) / scale_h)
coords.append((coord_x, coord_y))
result.append({'keypoints': coords, 'kp_score': scores})
imgname = str(frame_index) + '.jpg'
result = {'imgname': imgname, 'result': result}
img = vis_frame(img_data, result)
if show_txt >= 0:
img = cv2.putText(img,
'ONLY ONE PERSON IS ALLOWED!', (40, 40),
fontFace=cv2.FONT_ITALIC,
fontScale=1,
color=(0, 0, 255),
thickness=3)
show_txt -= 1
else:
# print warning
img_txt = cv2.putText(img_data,
'ONLY ONE PERSON IS ALLOWED!', (40, 40),
fontFace=cv2.FONT_ITALIC,
fontScale=1,
def detect_main(im_name, orig_img, det_model, pose_model, opt):
args = opt
mode = args.mode
inp_dim = int(opt.inp_dim)
dim = orig_img.shape[1], orig_img.shape[0]
img_ = (letterbox_image(orig_img, (inp_dim, inp_dim)))
img_ = img_[:, :, ::-1].transpose((2, 0, 1)).copy()
img = torch.from_numpy(img_).float().div(255.0).unsqueeze(0)
img = [img]
orig_img = [orig_img]
im_name = [im_name]
im_dim_list = [dim]
# img.append(img_k)
# orig_img.append(orig_img_k)
# im_name.append(im_name_k)
# im_dim_list.append(im_dim_list_k)
with torch.no_grad():
# Human Detection
img = torch.cat(img)
im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2)
# im_dim_list_ = im_dim_list
# DetectionLoader
det_inp_dim = int(det_model.net_info['height'])
assert det_inp_dim % 32 == 0
assert det_inp_dim > 32
# res_n = 0
with torch.no_grad():
img = img.cuda()
prediction = det_model(img, CUDA=True) # a tensor
boxes_chair = get_box(prediction, det_inp_dim, im_dim_list,
opt.confidence, opt.num_classes, 56)
boxes_sofa = get_box(prediction, det_inp_dim, im_dim_list,
opt.confidence, opt.num_classes, 57)
boxes_bed = get_box(prediction, det_inp_dim, im_dim_list,
opt.confidence, opt.num_classes, 59)
dets = dynamic_write_results(prediction,
opt.confidence,
opt.num_classes,
0,
nms=True,
nms_conf=opt.nms_thesh)
if isinstance(dets, int) or dets.shape[0] == 0:
# cv2.imwrite('err_result/no_person/'+im_name[0][0:-4]+'_re.jpg', orig_img[0])
return []
dets = dets.cpu()
im_dim_list = torch.index_select(im_dim_list, 0, dets[:, 0].long())
scaling_factor = torch.min(det_inp_dim / im_dim_list, 1)[0].view(-1, 1)
# coordinate transfer
dets[:, [1, 3]] -= (det_inp_dim -
scaling_factor * im_dim_list[:, 0].view(-1, 1)) / 2
dets[:, [2, 4]] -= (det_inp_dim -
scaling_factor * im_dim_list[:, 1].view(-1, 1)) / 2
dets[:, 1:5] /= scaling_factor
for j in range(dets.shape[0]):
dets[j, [1, 3]] = torch.clamp(dets[j, [1, 3]], 0.0, im_dim_list[j,
0])
dets[j, [2, 4]] = torch.clamp(dets[j, [2, 4]], 0.0, im_dim_list[j,
1])
boxes = dets[:, 1:5]
scores = dets[:, 5:6]
boxes_k = boxes[dets[:, 0] == 0]
if isinstance(boxes_k, int) or boxes_k.shape[0] == 0:
boxes = None
scores = None
inps = None
pt1 = None
pt2 = None
else:
inps = torch.zeros(boxes_k.size(0), 3, opt.inputResH, opt.inputResW)
pt1 = torch.zeros(boxes_k.size(0), 2)
pt2 = torch.zeros(boxes_k.size(0), 2)
orig_img = orig_img[0]
im_name = im_name[0]
boxes = boxes_k
scores = scores[dets[:, 0] == 0]
# orig_img[k], im_name[k], boxes_k, scores[dets[:, 0] == k], inps, pt1, pt2
# DetectionProcess
with torch.no_grad():
if boxes is None or boxes.nelement() == 0:
pass
else:
inp = im_to_torch(cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB))
inps, pt1, pt2 = crop_from_dets(inp, boxes, inps, pt1, pt2)
# self.Q.put((inps, orig_img, im_name, boxes, scores, pt1, pt2))
batchSize = args.posebatch
# fall_res_all = []
for i in range(1):
with torch.no_grad():
if boxes is None or boxes.nelement() == 0:
# writer.save(None, None, None, None, None, orig_img, im_name.split('/')[-1])
# res_n = 0
continue
# Pose Estimation
datalen = inps.size(0)
leftover = 0
if (datalen) % batchSize:
leftover = 1
num_batches = datalen // batchSize + leftover
hm = []
for j in range(num_batches):
inps_j = inps[j * batchSize:min((j + 1) *
batchSize, datalen)].cuda()
hm_j = pose_model(inps_j)
hm.append(hm_j)
hm = torch.cat(hm)
hm = hm.cpu()
# writer.save(boxes, scores, hm, pt1, pt2, orig_img, im_name.split('/')[-1])
fall_res = []
keypoint_res = []
# fall_res.append(im_name.split('/')[-1])
if opt.matching:
preds = getMultiPeakPrediction(hm, pt1.numpy(), pt2.numpy(),
opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH,
opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {'imgname': im_name, 'result': result}
# img = orig_img
img = vis_frame(orig_img, result)
for human in result['result']:
keypoint = human['keypoints']
kp_scores = human['kp_score']
keypoint = keypoint.numpy()
xmax = max(keypoint[:, 0])
xmin = min(keypoint[:, 0])
ymax = max(keypoint[:, 1])
ymin = min(keypoint[:, 1])
box_hm = [xmin, ymin, xmax, ymax]
kp_num = 0
for i in range(len(kp_scores)):
if kp_scores[i] > 0.05:
kp_num += 1
if kp_num < 10:
# cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)
fall_res.append([False, xmin, ymin, xmax, ymax])
# print("kp_num:"+str(kp_num))
continue
overlap = []
for box in boxes_chair:
overlap.append(compute_overlap(box_hm, box))
for box in boxes_sofa:
overlap.append(compute_overlap(box_hm, box))
for box in boxes_bed:
overlap.append(compute_overlap(box_hm, box))
if len(overlap) > 0 and max(overlap) >= 0.6:
# res_n = 0
fall_res.append([False, xmin, ymin, xmax, ymax])
# cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)
# print("overlap:"+str(overlap))
continue
w = xmax - xmin
h = ymax - ymin
ratio = w / h
# distance = abs((keypoint[15][1] + keypoint[16][1]) / 2 - (keypoint[11][1] + keypoint[12][1]) / 2)
xhead = (keypoint[1][0] + keypoint[2][0] + keypoint[2][0] +
keypoint[3][0] + keypoint[4][0]) / 4
yhead = (keypoint[1][1] + keypoint[2][1] + keypoint[2][1] +
keypoint[3][1] + keypoint[4][1]) / 4
xfeet = (keypoint[15][0] + keypoint[16][0]) / 2
yfeet = (keypoint[15][1] + keypoint[16][1]) / 2
d_ear = (abs(keypoint[3][0] - keypoint[4][0])**2 +
abs(keypoint[3][1] - keypoint[4][1])**2)**0.5
r = (w**2 + h**2)**0.5 / d_ear
if kp_scores[3] > 0.05 and kp_scores[4] > 0.05 and r < 4:
fall_res.append([False, xmin, ymin, xmax, ymax])
# cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)
# print("r<4")
continue
# distance = abs((keypoint[15][1] + keypoint[16][1]) / 2 - (keypoint[11][1] + keypoint[12][1]) / 2)
# xhead_foot = abs(xfeet - xhead)
# yhead_foot = abs(yfeet - yhead)
# dhead_foot = (xhead_foot ** 2 + yhead_foot ** 2) ** 0.5
# ratio = yhead_foot / dhead_foot
if min(kp_scores[3], kp_scores[4], kp_scores[15],
kp_scores[16]) > 0.05 and yfeet < (keypoint[3][1] +
keypoint[4][1]) / 2:
# cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2)
# font = cv2.FONT_HERSHEY_SIMPLEX
# cv2.putText(img, 'Warning!Fall!', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 255, 0), 2)
fall_res.append([True, xmin, ymin, xmax, ymax])
keypoint_res.append(keypoint)
# res_n = 2
elif w / h >= 1.0:
# cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2)
# font = cv2.FONT_HERSHEY_SIMPLEX
# cv2.putText(img, 'Warning!Fall', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 0, 255), 2)
fall_res.append([True, xmin, ymin, xmax, ymax])
keypoint_res.append(keypoint)
# res_n = 1
else:
# cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2)
# print("normal")
fall_res.append([False, xmin, ymin, xmax, ymax])
# res_n = 0
# cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
'''
for box in boxes_chair:
cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (255, 255, 0), 2)
for box in boxes_sofa:
cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (0, 255, 255), 2)
for box in boxes_bed:
cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (255, 0, 255), 2)
cv2.imwrite('err_result/false/'+im_name[0:-4]+'_re.jpg', img)
'''
return keypoint_res
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img,
im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(
os.path.join(self.outputpath, self.dir_folder,
im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
# if opt.matching:
# preds = getMultiPeakPrediction(
# hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
# result = matching(boxes, scores.numpy(), preds)
# else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW,
opt.outputResH, opt.outputResW)
# (hm_data, pt1, pt2, 320, 256, 80, 64)
result = pose_nms(boxes, scores, preds_img, preds_scores)
# (bbox定位list, bbox评分list, 位姿定位list, 位姿评分list)
# result[ {'keypoints', 'kp_score', 'proposal_score'}, {---}, ...], pPose_nms.py line-114
result = {'imgname': im_name, 'result': result}
# self.final_result.append(result)
self.show_img = vis_frame(orig_img, result)
cv2.imwrite(
os.path.join(self.outputpath, self.dir_folder,
im_name), img)
# if opt.save_img or opt.save_video or opt.vis:
# img = vis_frame(orig_img, result)
# if opt.vis:
# cv2.imshow("AlphaPose Demo", img)
# cv2.waitKey(30)
# if opt.save_img:
# cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
# if opt.save_video:
# self.stream.write(img)
else:
# time.sleep(0.1)
pass
def update(self):
# keep looping infinitely
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, "vis", f"{im_name}.jpg"), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
if opt.matching:
preds = getMultiPeakPrediction(
hm_data,
pt1.numpy(),
pt2.numpy(),
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW,
)
result = matching(boxes, scores.numpy(), preds)
else:
preds_hm, preds_img, preds_scores = getPrediction(
hm_data,
pt1,
pt2,
opt.inputResH,
opt.inputResW,
opt.outputResH,
opt.outputResW,
)
result = pose_nms(boxes, scores, preds_img, preds_scores)
result = {"imgname": im_name, "result": result}
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis or self.result_handler:
# Draw poses on frame or exececute custom result_handler
if self.result_handler is None:
img = vis_frame(orig_img, result)
else:
img = self.result_handler(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, "vis", f"{im_name}.jpg"), img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
def update(self):
# keep looping infinitely
temp_kps=[]
while True:
# if the thread indicator variable is set, stop the
# thread
if self.stopped:
if self.save_video:
self.stream.release()
return
# otherwise, ensure the queue is not empty
if not self.Q.empty():
(boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get()
orig_img = np.array(orig_img, dtype=np.uint8)
if boxes is None:
if opt.save_img or opt.save_video or opt.vis:
img = orig_img
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
# location prediction (n, kp, 2) | score prediction (n, kp, 1)
preds_hm, preds_img, preds_scores = getPrediction(
hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW)
result = pose_nms(boxes, scores, preds_img, preds_scores)
pos = result[0]['keypoints'].unsqueeze(0).numpy()
pos = self.aligner.align_points(pos)[0]
pos = (pos[..., :2] - 129) / 255
pos = torch.FloatTensor(pos)
kp = torch.cat((pos, result[0]['kp_score']), 1)
kp = kp.unsqueeze(0)
if len(temp_kps) < 9:
kp = kp.reshape([1, -1]).cuda()
temp_kps.append(kp)
kp = kp.repeat(9, 1).reshape(1, -1)
outputs = self.pos_reg_model(kp)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
else:
kp = kp.cuda().reshape(1,-1)
temp_kps.append(kp)
temp_kps.pop(0)
_temp_kps = torch.cat(temp_kps)
_temp_kps.cuda()
_temp_kps = _temp_kps.reshape([1, -1])
outputs = self.pos_reg_model(_temp_kps)
_, preds = torch.max(outputs, 1)
classidx = preds.cpu()
result[0]['class'] = str(float(classidx))
# print(preds)
result = {
'imgname': im_name,
'result': result
}
self.result_Q.put((boxes, classidx))
self.final_result.append(result)
if opt.save_img or opt.save_video or opt.vis:
img = vis_frame(orig_img, result)
if opt.vis:
cv2.imshow("AlphaPose Demo", img)
cv2.waitKey(30)
if opt.save_img:
cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img)
if opt.save_video:
self.stream.write(img)
else:
time.sleep(0.1)
