def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: img = orig_img cv2.waitKey(0) if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2_imshow(img) cv2.waitKey(0) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'imgname': im_name, 'result': result} self.final_result.append(result) img = vis_frame(orig_img, result) cv2_imshow(img) cv2.waitKey(0) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) cv2_imshow(img) cv2.waitKey(0) if opt.vis: cv2_imshow(img) cv2.waitKey(0) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def update(self): while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: print('released video stream') self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): start_time = getTime() (boxes, scores, hm_data, pt1, pt2, orig_img, img_id, car_np) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) img = orig_img # print('processing ' , img_id) """ PERSON """ person_list = self.person.person_tracking(boxes, scores, hm_data, pt1, pt2, img_id) self.person.person_tracjectory(person_list) vis_frame(img, person_list) """ Car """ if opt.park: car_dest_list = self.vehicle.car_tracking(car_np, img_id) self.vehicle.car_trajectory(car_dest_list) self.vehicle.parking_detection(car_dest_list, img, img_id) elif opt.fight: self.person.fight_detection(person_list, img_id) cv2.putText(img, f'Frame: {str(img_id).ljust(4)}', (10,40) , cv2.FONT_HERSHEY_DUPLEX, 2, WHITE, 2) # point is left-bottom # self.fight_detection(person_list) # self.car_person_detection(car_dest_list, bbox_dets_list, img) ckpt_time, det_time = getTime(start_time) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(33) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def start(self): # start to get the right result for i in range(len(self.Q)): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q[i] orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None or boxes.nelement() == 0: result = { 'imgname': im_name, 'result': None, } self.final_result.append(result) if opt.save_video: self.stream.write(orig_img) else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) if len(result) == 0: result = { 'imgname': im_name, 'result': None, } else: result = { 'imgname': im_name, 'result': result, } self.final_result.append(result) if opt.save_video: img = orig_img if opt.vis and result['result'] is not None: img = vis_frame(orig_img, result) self.stream.write(img)
def cal_coord(pred_heatmaps, images_anno, args): final_result = [] for img_id in pred_heatmaps.keys(): heat_h, heat_w, n_kpoints = pred_heatmaps[img_id].shape scale_h, scale_w = heat_h / images_anno[img_id][ 'height'], heat_w / images_anno[img_id]['width'] coord = [] scores = [] result = [] for p_ind in range(n_kpoints): heat = pred_heatmaps[img_id][:, :, p_ind] scores.append(np.max(heat)) heat = gaussian_filter(heat, sigma=5) ind = np.unravel_index(np.argmax(heat), heat.shape) ind = pose_processing(heat, ind) coord_x = int((ind[1] + 1) / scale_w) coord_y = int((ind[0] + 1) / scale_h) coord.append((coord_x, coord_y)) result.append({'keypoints': coord, 'kp_score': scores}) result = { 'imgname': images_anno[img_id]['file_name'], 'result': result } if args.save_imgs: ori_img = cv2.imread( os.path.join(args.img_path, images_anno[img_id]['file_name'])) img = vis_frame(ori_img, result) cv2.imshow('demo', img) cv2.waitKey(0) cv2.destroyAllWindows() cv2.imwrite( os.path.join(args.outputpath, 'vis', images_anno[img_id]['file_name']), img) final_result.append(result) return final_result
def update(self): # keep looping infinitely while True: print("Q_writer:", self.Q.qsize()) # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: img = orig_img if opt.save_img or opt.save_video or opt.vis: if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'imgname': im_name, 'result': result} self.final_result.append(result) img, _ = vis_frame(orig_img, result) if opt.save_img or opt.save_video or opt.vis: if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) #img = img.astype(np.uint8) temp = cv2.imencode(".jpg", img)[1] icode = np.array(temp) se = icode.tostring() n = len(se) self.dll.mjpg_write(c_char_p(se), n) else: time.sleep(0.1)
def main(): # root directory data_root = '/media/sdc1/' # trainval is the main dataset folder for the processed videos out_splitted = os.path.join(data_root, 'trainval') for i, (dire, folds, fils) in enumerate(os.walk(out_splitted)): if i == 0: print( 'Main dataset directory: {}\n Subdirectories within it: {}\n'. format(dire, folds)) continue # run command on the frames if len(folds) == 0 and len(fils) > 0: # loop over video frames print('Reading frames from: {}'.format(dire)) # read the annotations specific to this folder p_json = os.path.join(dire.replace('trainval', 'trainval_anns')) anns = load_json(p_json) if len(anns) > 0: # read frames p_imgs = [ os.path.join(dire, frame_name) for frame_name in fils if frame_name in anns ] frames_name = [p.split('/')[-1] for p in p_imgs] imgs = [cv2.imread(p) for p in p_imgs] # create a list of black images shapes = [img.shape for img in imgs] empty = [np.zeros(shape, dtype=np.uint8) for shape in shapes] # apply annotation key_imgs = [ vis_frame(empty[i], anns[kd]) for i, kd in enumerate(frames_name) ] # concatenate img_cat = [ np.concatenate((img, key_imgs[i]), axis=1) for i, img in enumerate(imgs) ] p_out = os.path.join( dire.replace('trainval', 'keypoint_frames')) if not os.path.isdir(p_out): os.makedirs(p_out) # save result to disk for i, im in enumerate(img_cat): p = os.path.join(p_out, frames_name[i]) cv2.imwrite(p, im)
def forward(self, Q_det): clock = 0 while 1: with torch.no_grad(): start_time = getTime() (inps, orig_img, boxes, scores, pt1, pt2) = Q_det.get() ckpt_time, det_time = getTime(start_time) img = orig_img if boxes is not None: hm_data = self.pose_model(inps.cuda()) ckpt_time, pose_time = getTime(ckpt_time) hm_data = hm_data.cpu() preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'result': result} img, datas, valid = vis_frame(img, result, opt.webcam_num) ckpt_time, post_time = getTime(ckpt_time) #print("det_time={:.3f}, pose_time={:.3f}, post_time={:.3f}, fps={:.1f}".format(det_time,pose_time,post_time, 1/(det_time+pose_time+post_time))) if valid: for data in datas: clock = 0 self.mq.sendMsg(data) print(data) else: clock = clock + 1 if clock > 500: self.mq.sendMsg( '{{ "source": "Behavior", "deviceId": {}, "hand": "{},{}", "body": {}, "state": {} }}' .format(0, 0, 0, 0, 0)) clock = 0 else: time.sleep(0.05) clock = clock + 1 if clock > 500: self.mq.sendMsg( '{{ "source": "Behavior", "deviceId": {}, "hand": "{},{}", "body": {}, "state": {} }}' .format(0, 0, 0, 0, 0)) clock = 0 '''' self.mq.sendMsg('{ "source": "Behavior", "deviceId": 0, "hand": "0,0", "body": 0 }') print('{ "source": "Behavior", "deviceId": 0, "hand": "0,0", "body": 0 }') ''' if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(10) temp = cv2.imencode(".jpg", img)[1] icode = np.array(temp) se = icode.tostring() n = len(se) self.dll.mjpg_write(c_char_p(se), n)
def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW, opt.nClasses) # print('dataloader:',boxes.shape, scores.shape, preds_img.shape, preds_scores.shape) result = pose_nms(boxes, scores, preds_img, preds_scores) result = { 'imgname': im_name, 'result': result, 'boxes':boxes } # print('dataloader.py:result:',result) # print('dataloader.py:hm_data.shape:',hm_data.shape) # print('dataloader.py:preds_hm.shape:',preds_hm.shape) # self.count+=1 # print('dataloader.py:count:',self.count) self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) if opt.vis and len(img)!=0: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img and len(img)!=0: cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video and len(img)!=0: self.stream.write(img) else: time.sleep(0.1)
def update(boxes, scores, hm_data, pt1, pt2, orig_img, i): orig_img = np.array(orig_img, dtype=np.uint8) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms( boxes, scores, preds_img, preds_scores) result = { 'imgname': str(i), 'result': result } img = vis_frame(orig_img, result) return img, result
def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (result, orig_img, im_name) = self.Q.get() if result is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: import pickle with open("result.pkl", "wb") as f: pickle.dump(result, f) self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.05)
def cal_coord(pred_heatmaps, images_anno, args, predictions): final_result = [] for img_id in pred_heatmaps.keys(): heat_h, heat_w, n_kpoints = pred_heatmaps[img_id].shape scale_h, scale_w = heat_h / images_anno[img_id][ 'height'], heat_w / images_anno[img_id]['width'] coord = [] scores = [] result = [] for p_ind in range(n_kpoints): heat = pred_heatmaps[img_id][:, :, p_ind] scores.append(np.max(heat)) heat = gaussian_filter(heat, sigma=5) ind = np.unravel_index(np.argmax(heat), heat.shape) ind = pose_processing(heat, ind) coord_x = int((ind[1] + 1) / scale_w) coord_y = int((ind[0] + 1) / scale_h) coord.append((coord_x, coord_y)) result.append({'keypoints': coord, 'kp_score': scores}) result = { 'imgname': images_anno[img_id]['file_name'], 'result': result } if args.save_imgs: if args.dataset_to_use == 'COCO': ori_img = cv2.imread( os.path.join(os.path.join(args.coco_dir, 'val2017'), images_anno[img_id]['file_name'])) else: ori_img = cv2.imread( os.path.join(args.img_path, images_anno[img_id]['file_name'])) img = vis_frame(ori_img, result) img_name = images_anno[img_id]['file_name'] ind = img_name.find('.') img_saving_name = img_name[:ind] + '{:s}'.format( 'T' if predictions[img_id] else 'F') + img_name[ind:] cv2.imwrite(os.path.join(args.outputpath, img_saving_name), img) final_result.append(result) return final_result
def gen_pose(self): with torch.no_grad(): (inps, orig_img, boxes, scores, pt1, pt2) = self.human_detect_result if boxes is None or boxes.nelement() == 0: self.det_human_num = 0 self.out_img = orig_img self.result = None self.resultNew = None return #print(inps, 'inps') #print(type(boxes), 'boxes') # inps = inps.cuda() hm = self.pose_model(inps) hm = [hm] hm = torch.cat(hm) hm = hm.cpu() hm_data = hm orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: self.det_human_num = 0 self.out_img = orig_img self.result = None self.resultNew = None return else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) self.result = result self.resultNew = conserv_result_format_to_list(self.result) resultDict = {'imgname': 'zhjs', 'result': result} self.out_img = vis_frame(orig_img, resultDict) self.det_human_num = len(result)
def get_pose(self, img_names): if len(img_names) > 1: start_lc = 4000 start_rc = 4000 now_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()) print('========START-Ten========') final_result = [] vis_images = [] height_difference = [] for img_index in range(len(img_names)): print('--------------------') img_name = img_names[img_index] try: img, orig_img, im_name, im_dim_list = [], [], [], [] inp_dim = int(self.args.inp_dim) im_name_k = img_name img_k, orig_img_k, im_dim_list_k = prep_image( im_name_k, inp_dim) img.append(img_k) orig_img.append(orig_img_k) im_name.append(im_name_k) im_dim_list.append(im_dim_list_k) except: print('index-{}: image have problem'.format(img_index)) final_result.append((None, None)) continue with torch.no_grad(): img = torch.cat(img) im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2) img = img.cuda() prediction = self.det_model(img, CUDA=True) dets = dynamic_write_results(prediction, self.args.confidence, self.args.num_classes, nms=True, nms_conf=self.args.nms_thesh) if isinstance(dets, int) or dets.shape[0] == 0: print('index-{}: No person detected'.format(img_index)) final_result.append((None, None)) height_difference.append(None) continue dets = dets.cpu() im_dim_list = torch.index_select(im_dim_list, 0, dets[:, 0].long()) scaling_factor = torch.min(self.det_inp_dim / im_dim_list, 1)[0].view(-1, 1) dets[:, [1, 3]] -= (self.det_inp_dim - scaling_factor * im_dim_list[:, 0].view(-1, 1)) / 2 dets[:, [2, 4]] -= (self.det_inp_dim - scaling_factor * im_dim_list[:, 1].view(-1, 1)) / 2 dets[:, 1:5] /= scaling_factor for j in range(dets.shape[0]): dets[j, [1, 3]] = torch.clamp(dets[j, [1, 3]], 0.0, im_dim_list[j, 0]) dets[j, [2, 4]] = torch.clamp(dets[j, [2, 4]], 0.0, im_dim_list[j, 1]) boxes = dets[:, 1:5] scores = dets[:, 5:6] k = 0 boxes_k = boxes[dets[:, 0] == k] inps = torch.zeros(boxes_k.size(0), 3, self.args.inputResH, self.args.inputResW) pt1 = torch.zeros(boxes_k.size(0), 2) pt2 = torch.zeros(boxes_k.size(0), 2) orig_img, im_name, boxes, scores, inps, pt1, pt2 = orig_img[ k], im_name[k], boxes_k, scores[dets[:, 0] == k], inps, pt1, pt2 inp = im_to_torch(cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB)) inps, pt1, pt2 = crop_from_dets(inp, boxes, inps, pt1, pt2) batchSize = self.args.posebatch datalen = inps.size(0) leftover = 0 if (datalen) % batchSize: leftover = 1 num_batches = datalen // batchSize + leftover hm = [] for j in range(num_batches): inps_j = inps[j * batchSize:min( (j + 1) * batchSize, datalen)].cuda() hm_j = self.pose_model(inps_j) hm.append(hm_j) hm = torch.cat(hm) hm_data = hm.cpu() orig_img = np.array(orig_img, dtype=np.uint8) im_name = im_name.split('/')[-1] preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, self.args.inputResH, self.args.inputResW, self.args.outputResH, self.args.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'imgname': im_name, 'result': result} img = vis_frame(orig_img, result) vis_images.append(img) outpur_dir = os.path.join(self.args.outputpath, 'vis') outpur_dir_raw = os.path.join(self.args.outputpath, 'raw') if not os.path.exists(outpur_dir): os.makedirs(outpur_dir) if not os.path.exists(outpur_dir_raw): os.makedirs(outpur_dir_raw) width = img.shape[1] keypoints = [res['keypoints'][0] for res in result['result']] distance = [xy[0] - width / 2 for xy in keypoints] distance = torch.tensor([torch.abs(m) for m in distance]) indice = torch.argsort(distance)[0] pose_result = result['result'][indice]['keypoints'] # left_arm = pose_result[[6, 8, 10]].numpy() # right_arm = pose_result[[5, 7, 9]].numpy() # ['Nose', 'LEye', 'REye', 'LEar', 'REar', 'LShoulder', 'RShoulder', 'LElbow', 'RElbow', 'LWrist', 'RWrist', 'LHip', # 'RHip', 'LKnee', 'RKnee', 'LAnkle', 'RAnkle'] left_arm = pose_result[[10]].numpy().astype(int) right_arm = pose_result[[9]].numpy().astype(int) left_arm_c_y = np.mean(left_arm, axis=0)[1] right_arm_c_y = np.mean(right_arm, axis=0)[1] # left_arm_c = tuple(np.mean(left_arm, axis=0).astype(int)) # right_arm_c = tuple(np.mean(right_arm, axis=0).astype(int)) left_arm_c = tuple(left_arm[0]) right_arm_c = tuple(right_arm[0]) hd = np.abs(left_arm_c_y - right_arm_c_y) height_difference.append(hd) cv2.circle(img, left_arm_c, 10, (0, 255, 0), -1, 8) cv2.circle(img, right_arm_c, 10, (0, 255, 0), -1, 8) log__vis_name = now_time + '-' + im_name cv2.imwrite(os.path.join(outpur_dir_raw, log__vis_name), orig_img) cv2.imwrite(os.path.join(outpur_dir, log__vis_name), img) if start_lc == 4000 and start_rc == 4000: start_lc = left_arm_c_y start_rc = right_arm_c_y left_move = 0 right_move = 0 else: left_move = left_arm_c_y - start_lc right_move = right_arm_c_y - start_rc print('index-{}--{}: left_c {:0f},right_c {:0f}'.format( img_index, im_name, left_arm_c_y, right_arm_c_y)) print('index-{}--{}: start_lc {:0f},start_rc {:0f}'.format( img_index, im_name, start_lc, start_rc)) print('index-{}--{}: left_move {:0f},right_move {:0f}'.format( img_index, im_name, left_move, right_move)) print('index-{}--{}: height_difference {:0f}'.format( img_index, im_name, hd)) final_result.append((left_move, right_move)) return final_result, vis_images, now_time, height_difference elif len(img_names) == 1: now_time = time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()) print('========START-One========') final_result = [] vis_images = [] height_difference = [] for img_index in range(len(img_names)): img_name = img_names[img_index] try: img, orig_img, im_name, im_dim_list = [], [], [], [] inp_dim = int(self.args.inp_dim) im_name_k = img_name img_k, orig_img_k, im_dim_list_k = prep_image( im_name_k, inp_dim) img.append(img_k) orig_img.append(orig_img_k) im_name.append(im_name_k) im_dim_list.append(im_dim_list_k) except: print('index-{}: image have problem'.format(img_index)) final_result.append((None, None)) with torch.no_grad(): img = torch.cat(img) vis_img = img.numpy()[0] vis_img = np.transpose(vis_img, (1, 2, 0)) vis_img = vis_img[:, :, ::-1] vis_images.append(vis_img) im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2) img = img.cuda() prediction = self.det_model(img, CUDA=True) dets = dynamic_write_results(prediction, self.args.confidence, self.args.num_classes, nms=True, nms_conf=self.args.nms_thesh) if isinstance(dets, int) or dets.shape[0] == 0: print('index-{}: No person detected'.format(img_index)) final_result.append((None, None)) else: print('index-{}: Person detected'.format(img_index)) final_result.append((4, 4)) return final_result, vis_images, now_time, height_difference
def update(self): next_id = 0 while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): start_time = getTime() (boxes, scores, hm_data, pt1, pt2, orig_img, img_id, CAR) = self.Q.get() # print(img_id) orig_img = np.array(orig_img, dtype=np.uint8) img = orig_img # text_filled2(img,(5,200),str(img_id),LIGHT_GREEN,2,2) """ PERSON """ if boxes is None: # No person detection person_list = [] pass else: person_list = self.person_tracking(boxes, scores, hm_data, pt1, pt2, img_id) # update frame vis_frame(img, person_list) """ Car """ if CAR is not None: car_np = CAR car_dest_list = self.car_tracking(car_np, img_id) else: car_dest_list = [] self.car_list_list.append(car_dest_list) self.person_list_list.append(person_list) self.car_trajectory(car_dest_list) self.person_tracjectory(person_list) # FOR GIST2019 if img_id != 0: self.fight_detection(person_list) # self.parking_detection(car_dest_list, img, img_id) # self.car_person_detection(car_dest_list, bbox_dets_list, img) # FOR NEXPA # self.person_nexpa(bbox_dets_list,img,img_id) ckpt_time, det_time = getTime(start_time) # cv2.putText(img, str(1 / det_time), (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1) cv2.putText(img, str(img_id), (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(33) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def update(self): count = 0 # filepath = '/home/yurik/Documents/Program/Alphapose_zed_video/testdata/20191014/walkstraightly/walkstraightly.svo' # init = sl.InitParameters(svo_input_filename=filepath,svo_real_time_mode=False) # init.depth_mode = sl.DEPTH_MODE.DEPTH_MODE_QUALITY # cam = sl.Camera() # runtime = sl.RuntimeParameters() # status = cam.open(init) # mat = sl.Mat() # zeroarr = np.zeros((720,1280,3)) # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) if opt.matching: preds = getMultiPeakPrediction( hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms( boxes, scores, preds_img, preds_scores) result = { 'imgname': im_name, 'result': result } # 3D coordinates computation ppl = result['result'] ppl_num = len(ppl) self.coordinates_u, self.coordinates_v, self.truex, self.truey, self.dists = fl.people_3d_coord(ppl, ppl_num, self.video_mode, self.camMtx1, orig_img) self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) # err = cam.grab(runtime) # if err == sl.ERROR_CODE.SUCCESS: # cam.retrieve_image(mat, sl.VIEW.VIEW_DEPTH) # depthmap = mat.get_data() # if img.shape[2] == 3: # depthmap = cv2.cvtColor(depthmap, cv2.COLOR_RGBA2RGB) # depthmap = cv2.resize(depthmap, (int(img.shape[1]/2), img.shape[0])) # depthmap = cv2.applyColorMap(depthmap, cv2.COLORMAP_JET) # depthmap = np.hstack((depthmap, zeroarr)) # depthmap = depthmap.astype(np.uint8) # img = cv2.addWeighted(img, 0.5, depthmap, 0.5, 3) if len(self.coordinates_v) > 0 and len(self.coordinates_u) > 0: for i in range(len(self.coordinates_v)): # cv2.putText(img, 'z:' + str(round((self.dists[i] / 10), 1)), # (int(self.coordinates_u[i]), int(self.coordinates_v[i]) - 15), # cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 3, 8) cv2.putText(img, str(round((self.truex[i] / 10), 1)), (int(self.coordinates_u[i]), int(self.coordinates_v[i]) - 15), cv2.FONT_HERSHEY_PLAIN, 2, (0, 255, 0), 3, 8) cv2.putText(img, str(round((self.truey[i] / 10), 1)), (int(self.coordinates_u[i]) + 200, int(self.coordinates_v[i]) - 15), cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 3, 8) cv2.putText(img, str(round((self.dists[i] / 10), 1)), (int(self.coordinates_u[i]) + 400, int(self.coordinates_v[i]) - 15), cv2.FONT_HERSHEY_PLAIN, 2, (0, 0, 255), 3, 8) cv2.putText(img, 'frames: ' + str(count), (620, 620), cv2.FONT_HERSHEY_PLAIN, 2, (0,100,90), 3, 8) else: cv2.putText(img, '[N/A]', (40, 620), cv2.FONT_HERSHEY_PLAIN, 2, (0, 100, 90), 3, 8) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1) count = count + 1
keypoints = d[i]['keypoints'] dim = len(keypoints) kp_scores = [] for i in range(len(keypoints) - 1, 0, -3): kp_scores.append(keypoints.pop(i)) kp_scores.reverse() kp_scores = torch.FloatTensor(kp_scores) kp_preds = np.reshape(keypoints, (int(dim / 3), 2)) # kp_preds = list(kp_preds) kp_preds = torch.FloatTensor(kp_preds) human = {'keypoints': kp_preds, 'kp_score': kp_scores} result.append(human) all_result = {'imgname': d[0]['image_id'], 'result': result} # print(all_result['result']) img = fn.vis_frame(image, all_result) cv2.imshow('img', img) cv2.waitKey(0) print('//////////////////////////////////') # kp_scores = [] # for i in range(len(keypoints)-1, 0, -3): # kp_scores.append( keypoints.pop(i) ) # kp_scores.reverse() # kp_preds = np.reshape(keypoints,(17,2)) # human = {'keypoints':kp_preds, 'kp_score':kp_scores} print('///////////////////////') # print(human['keypoints'])
def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: ''' with open("IP2.csv",'a',newline='') as t: writer=csv.writer(t) writer.writerow([0]) ''' if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(0) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) if opt.matching: preds = getMultiPeakPrediction(hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'imgname': im_name, 'result': result} ''' with open("IP2.csv",'a',newline='') as t: writer=csv.writer(t) writer.writerow([data]) ''' self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img, data = vis_frame(orig_img, result) #img = vis_frame(orig_img, result) #draw human pose limbs and keypoints if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def update(self): # keep looping infinitely temp_kps = [] while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: h, w, c = img.shape img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC) cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) #print(len(result)) #print(boxes.shape) result = [] if preds_img.shape[0] > 0: for re in range(preds_img.shape[0]): pos = preds_img[re].unsqueeze(0).numpy() pos = self.aligner.align_points(pos)[0] pos = (pos[..., :2] - 129) / 255 pos = torch.FloatTensor(pos) kp = torch.cat( (pos, preds_scores[re].unsqueeze(1)), 1) kp = kp.unsqueeze(0) kp = kp.reshape([1, -1]).cuda() kp = kp.repeat(9, 1).reshape(1, -1) outputs = self.pos_reg_model(kp) _, preds = torch.max(outputs, 1) classidx = preds.cpu() result.append({ 'class': str(float(classidx)), 'keypoints': preds_img[re], 'kp_score': preds_scores[re].unsqueeze(1), 'bbox': boxes[re] }) # print(preds) result = {'imgname': im_name, 'result': result} # if len(result)>0: # for re in range(len(result)): # pos = result[re]['keypoints'].unsqueeze(0).numpy() # pos = self.aligner.align_points(pos)[0] # pos = (pos[..., :2] - 129) / 255 # pos = torch.FloatTensor(pos) # kp = torch.cat((pos, result[0]['kp_score']), 1) # kp = kp.unsqueeze(0) # kp = kp.reshape([1, -1]).cuda() # kp = kp.repeat(9, 1).reshape(1, -1) # outputs = self.pos_reg_model(kp) # _, preds = torch.max(outputs, 1) # classidx = preds.cpu() # result[re]['class'] = str(float(classidx)) # # print(preds) # result = { # 'imgname': im_name, # 'result': result # } self.result_Q.put(result) self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) if opt.vis: h, w, c = img.shape img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC) cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) # 发送图像 img = vis_frame(orig_img, result) h, w, c = img.shape img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC) self.tcp_client.send_img(img) else: time.sleep(0.1)
size = ((int(device.get(cv2.CAP_PROP_FRAME_WIDTH)),int(device.get(cv2.CAP_PROP_FRAME_HEIGHT)))) success, frame = device.read() ################################################################################### success = True frame_id = 0 while success: success, frame = device.read() img_final_list = find_athlete(test[frame_id], my_matrix, tennis_width, tennis_height) # frame_queue.put(frame) frame_id = frame_id + 1 img_final_list = find_athlete(test[150], my_matrix, tennis_width, tennis_height) img = fn.vis_frame(image,test[150]) # cv2.imshow('img', img) # cv2.waitKey(0) ########################################################### # resize target image. Don't care this part: court = cv2.imread("court|.png") # court = court[140:705, 160:390, :] # range of height and width court = court[140:705, 164:387, :] scale = (tennis_width, tennis_height) court = cv2.resize(court, scale) ########################################################### # cv2.circle(court, people_transf_position, 3, (0,0,0),10)
def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = { 'imgname': im_name, 'result': result } # append imgname here. # result here includes imgname, bbox, kps, kp_score, proposal_score # Critical, run pnp algorithm here to get 6d pose. # embed() if result['result']: kp_2d = np.array(result['result'][0]['keypoints']) kp_3d = np.array(self.kp_3d) R, t = pnp(kp_3d, kp_2d, self.cam_K) result.update({'cam_R': R, 'cam_t': t}) else: result.update({'cam_R': [], 'cam_t': []}) self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def update(self): next_id = 0 car_next_id = 0 bbox_dets_list_list = [] keypoints_list_list = [] car_dets_list_list = [] while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): start_time = getTime() (boxes, scores, hm_data, pt1, pt2, orig_img, img_id, CAR) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is not None: boxes = boxes.astype(np.int32) img = orig_img # text_filled2(img,(5,200),str(img_id),LIGHT_GREEN,2,2) bbox_dets_list = [] # keyframe: start from empty keypoints_list = [] # keyframe: start from empty # print(boxes) if boxes is None: # No person detection pass # bbox_det_dict = {"img_id": img_id, # "det_id": 0, # "track_id": None, # "bbox": [0, 0, 2, 2]} # bbox_dets_list.append(bbox_det_dict) # # keypoints_dict = {"img_id": img_id, # "det_id": 0, # "track_id": None, # "keypoints": []} # keypoints_list.append(keypoints_dict) else: if opt.matching: preds = getMultiPeakPrediction( hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction(hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) # print('number of result', preds_hm, preds_scores ) result = pose_nms(boxes, scores, preds_img, preds_scores) # list type # result = { 'keypoints': , 'kp_score': , 'proposal_score': , 'bbox' } if img_id > 0: # First frame does not have previous frame bbox_list_prev_frame = bbox_dets_list_list[img_id - 1].copy() keypoints_list_prev_frame = keypoints_list_list[img_id - 1].copy() else: bbox_list_prev_frame = [] keypoints_list_prev_frame = [] # boxes.size(0) num_dets = len(result) for bbox in boxes: x, y, w, h = bbox.astype(np.uint32) cv2.rectangle(orig_img, (x, y), (x + w, y + h), (253, 222, 111), 1) for det_id in range(num_dets): # IOU tracking for detections in current frame. # detections for current frame # obtain bbox position and track id result_box = result[det_id] kp_score = result_box['kp_score'] proposal_score = result_box['proposal_score'].numpy()[0] if proposal_score < 1.3: continue keypoints = result_box['keypoints'] # torch, (17,2) keypoints_pf = np.zeros((15, 2)) idx_list = [16, 14, 12, 11, 13, 15, 10, 8, 6, 5, 7, 9, 0, 0, 0] for i, idx in enumerate(idx_list): keypoints_pf[i] = keypoints[idx] keypoints_pf[12] = (keypoints[5] + keypoints[6]) / 2 # neck # COCO-order {0-nose 1-Leye 2-Reye 3-Lear 4Rear 5-Lsho 6-Rsho 7-Lelb 8-Relb 9-Lwri 10-Rwri 11-Lhip 12-Rhip 13-Lkne 14-Rkne 15-Lank 16-Rank} # PoseFLow order #{0-Rank 1-Rkne 2-Rhip 3-Lhip 4-Lkne 5-Lank 6-Rwri 7-Relb 8-Rsho 9-Lsho 10-Lelb 11-Lwri 12-neck 13-nose 14-TopHead} bbox_det = bbox_from_keypoints(keypoints) # xxyy # bbox_in_xywh = enlarge_bbox(bbox_det, enlarge_scale) # bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh) # Keyframe: use provided bbox # if bbox_invalid(bbox_det): # track_id = None # this id means null # keypoints = [] # bbox_det = [0, 0, 2, 2] # # update current frame bbox # bbox_det_dict = {"img_id": img_id, # "det_id": det_id, # "track_id": track_id, # "bbox": bbox_det} # bbox_dets_list.append(bbox_det_dict) # # update current frame keypoints # keypoints_dict = {"img_id": img_id, # "det_id": det_id, # "track_id": track_id, # "keypoints": keypoints} # keypoints_list.append(keypoints_dict) # continue # # update current frame bbox if img_id == 0: # First frame, all ids are assigned automatically track_id = next_id next_id += 1 else: track_id, match_index = get_track_id_SpatialConsistency(bbox_det, bbox_list_prev_frame) # print('track' ,track_id, match_index) if track_id != -1: # if candidate from prev frame matched, prevent it from matching another del bbox_list_prev_frame[match_index] del keypoints_list_prev_frame[match_index] # update current frame bbox bbox_det_dict = {"img_id": img_id, "det_id": det_id, "track_id": track_id, "bbox": bbox_det} # update current frame keypoints keypoints_dict = {"img_id": img_id, "det_id": det_id, "track_id": track_id, "keypoints": keypoints, 'kp_poseflow': keypoints_pf, 'kp_score': kp_score, 'bbox': bbox_det, 'proposal_score': proposal_score} bbox_dets_list.append(bbox_det_dict) keypoints_list.append(keypoints_dict) num_dets = len(bbox_dets_list) for det_id in range(num_dets): # if IOU tracking failed, run pose matching tracking. bbox_det_dict = bbox_dets_list[det_id] keypoints_dict = keypoints_list[det_id] # assert (det_id == bbox_det_dict["det_id"]) # assert (det_id == keypoints_dict["det_id"]) if bbox_det_dict["track_id"] == -1: # this id means matching not found yet # track_id = bbox_det_dict["track_id"] track_id, match_index = get_track_id_SGCN(bbox_det_dict["bbox"], bbox_list_prev_frame, keypoints_dict["kp_poseflow"], keypoints_list_prev_frame) if track_id != -1: # if candidate from prev frame matched, prevent it from matching another del bbox_list_prev_frame[match_index] del keypoints_list_prev_frame[match_index] bbox_det_dict["track_id"] = track_id keypoints_dict["track_id"] = track_id # if still can not find a match from previous frame, then assign a new id # if track_id == -1 and not bbox_invalid(bbox_det_dict["bbox"]): if track_id == -1: bbox_det_dict["track_id"] = next_id keypoints_dict["track_id"] = next_id next_id += 1 # update frame # print('keypoint list', len(keypoints_list)) vis_frame(img, keypoints_list) """ Car """ if CAR is not None: car_np = CAR new_car_bboxs = car_np[:, 0:4].astype(np.uint32) # b/ x y w h c / cls_conf, cls_idx new_car_score = car_np[:, 4] cls_conf = car_np[:, 4] # print("id: ", img_id , " ------------ " , new_car_bboxs, new_car_score) # cls_conf = car_np[:, 6] car_dest_list = [] if img_id > 1: # First frame does not have previous frame car_bbox_list_prev_frame = car_dets_list_list[img_id - 1].copy() else: car_bbox_list_prev_frame = [] # print('car bbox list prev frame ', len(car_bbox_list_prev_frame)) for c, score, conf in zip(new_car_bboxs, new_car_score, cls_conf): # car_bbox_det = c # car_bbox_det = x1y1x2y2_to_xywh(c) bbox_det = c # bbox_in_xywh = enlarge_bbox(car_bbox_det, enlarge_scale) # bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh) if img_id == 0: # First frame, all ids are assigned automatically car_track_id = car_next_id car_next_id += 1 else: car_track_id, match_index = get_track_id_SpatialConsistency(bbox_det, car_bbox_list_prev_frame) # print(car_track_id, match_index) if car_track_id != -1: # if candidate from prev frame matched, prevent it from matching another del car_bbox_list_prev_frame[match_index] bbox_det_dict = {"img_id": img_id, "track_id": car_track_id, "bbox": bbox_det, "score": score, "conf": conf} car_dest_list.append(bbox_det_dict) for car_bbox_det_dict in car_dest_list: # detections for current frame if car_bbox_det_dict["track_id"] == -1: # this id means matching not found yet car_bbox_det_dict["track_id"] = car_next_id car_next_id += 1 self.tracking(car_dest_list) car_dets_list_list.append(car_dest_list) else: car_dest_list = [] bbox_det_dict = {"img_id": img_id, "det_id": 0, "track_id": None, "bbox": [0, 0, 2, 2], "score": 0, "conf": 0} car_dest_list.append(bbox_det_dict) car_dets_list_list.append(car_dest_list) bbox_dets_list_list.append(bbox_dets_list) keypoints_list_list.append(keypoints_list) if img_id != 0: self.car_person_detection(car_dest_list, bbox_dets_list, img) self.car_parking_detection(car_dest_list, img, img_id) ckpt_time, det_time = getTime(start_time) cv2.putText(img, str(1 / det_time), (5, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 1) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(33) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def update(self): print(f'DataWriter_update_thread: {threading.currentThread().name}') # keep looping infinitely temp_kps = [] while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: h, w, c = img.shape img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC) cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) # 发送图像 img = orig_img h, w, c = img.shape img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC) # 绘制床的位置矩形 # cv2.rectangle(img, (conf.Urls.bed_min_x, conf.Urls.bed_min_y), # (conf.Urls.bed_max_x, conf.Urls.bed_max_y), (0, 255, 0), 1) self.tcp_client.send_img(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) if len(result) > 0: pos = result[0]['keypoints'].unsqueeze(0).numpy() pos = self.aligner.align_points(pos)[0] pos = (pos[..., :2] - 129) / 255 pos = torch.FloatTensor(pos) kp = torch.cat((pos, result[0]['kp_score']), 1) kp = kp.unsqueeze(0) if len(temp_kps) < 9: kp = kp.reshape([1, -1]).cuda() temp_kps.append(kp) kp = kp.repeat(9, 1).reshape(1, -1) outputs = self.pos_reg_model(kp) _, preds = torch.max(outputs, 1) classidx = preds.cpu() result[0]['class'] = str(float(classidx)) result[0]['bbox'] = boxes[0] else: kp = kp.cuda().reshape(1, -1) temp_kps.append(kp) temp_kps.pop(0) _temp_kps = torch.cat(temp_kps) _temp_kps.cuda() _temp_kps = _temp_kps.reshape([1, -1]) outputs = self.pos_reg_model(_temp_kps) _, preds = torch.max(outputs, 1) classidx = preds.cpu() result[0]['class'] = str(float(classidx)) result[0]['bbox'] = boxes[0] # print(preds) result = {'imgname': im_name, 'result': result} self.result_Q.put(result) self.final_result.append(result) # 发送图像 img = vis_frame(orig_img, result) # h, w, c = img.shape # img = cv2.resize(img, (int(w / 2), int(h / 2)), interpolation=cv2.INTER_CUBIC) # 绘制床的位置矩形 # cv2.rectangle(img, (conf.Urls.bed_min_x, conf.Urls.bed_min_y), (conf.Urls.bed_max_x, conf.Urls.bed_max_y), (0, 255, 0), 1) self.tcp_client.send_img(img) if opt.save_img or opt.save_video or opt.vis: # img = vis_frame(orig_img, result) if opt.vis: h, w, c = img.shape #img = cv2.resize(img, (int(w / 4), int(h / 4)), interpolation=cv2.INTER_CUBIC) cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def update(self): # keep looping infinitely frame_prev = -1 frame_cur = 0 img_id = -1 next_id = 0 bbox_dets_list_list = [] keypoints_list_list = [] car_dets_list_list = [] car_next_id = 0 while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, img_id, CAR) = self.Q.get() # print(img_id) orig_img = np.array(orig_img, dtype=np.uint8) img = orig_img bbox_dets_list = [] # keyframe: start from empty keypoints_list = [] # keyframe: start from empty if boxes is None: # No person detection bbox_det_dict = { "img_id": img_id, "det_id": 0, "track_id": None, "bbox": [0, 0, 2, 2] } bbox_dets_list.append(bbox_det_dict) keypoints_dict = { "img_id": img_id, "det_id": 0, "track_id": None, "keypoints": [] } keypoints_list.append(keypoints_dict) bbox_dets_list_list.append(bbox_dets_list) keypoints_list_list.append(keypoints_list) else: if opt.matching: preds = getMultiPeakPrediction(hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) # list type # 'keypoints': # 'kp_score': # 'proposal_score': # 'bbox' # # print('boexes', boxes.size(), boxes) # for aa in result: # keys = aa['keypoints'] # bbox2 = aa['bbox'] # print('pose nms keys', keys.size()) # print('pose nms, box', bbox2.size(), bbox2) # # _result = { # 'imgname': img_id, # 'result': result, # 'pt1': pt1, # 'pt2': pt2 # } if img_id > 0: # First frame does not have previous frame bbox_list_prev_frame = bbox_dets_list_list[img_id - 1].copy() keypoints_list_prev_frame = keypoints_list_list[ img_id - 1].copy() else: bbox_list_prev_frame = [] keypoints_list_prev_frame = [] # boxes.size(0) num_dets = len(result) for det_id in range( num_dets): # detections for current frame # obtain bbox position and track id result_box = result[det_id] kp_score = result_box['kp_score'] proposal_score = result_box['proposal_score'].numpy( )[0] if proposal_score < 1.3: continue keypoints = result_box['keypoints'] bbox_det = bbox_from_keypoints(keypoints) # xxyy # enlarge bbox by 20% with same center position # bbox_x1y1x2y2 = xywh_to_x1y1x2y2(bbox_det) bbox_in_xywh = enlarge_bbox(bbox_det, enlarge_scale) # print('enlared', bbox_in_xywh) bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh) # print('converted', bbox_det) # Keyframe: use provided bbox # if bbox_invalid(bbox_det): # track_id = None # this id means null # keypoints = [] # bbox_det = [0, 0, 2, 2] # # update current frame bbox # bbox_det_dict = {"img_id": img_id, # "det_id": det_id, # "track_id": track_id, # "bbox": bbox_det} # bbox_dets_list.append(bbox_det_dict) # # update current frame keypoints # keypoints_dict = {"img_id": img_id, # "det_id": det_id, # "track_id": track_id, # "keypoints": keypoints} # keypoints_list.append(keypoints_dict) # continue # # update current frame bbox # obtain keypoints for each bbox position in the keyframe # print('img id ', img_id) if img_id == 0: # First frame, all ids are assigned automatically track_id = next_id next_id += 1 else: track_id, match_index = get_track_id_SpatialConsistency( bbox_det, bbox_list_prev_frame) # print('track' ,track_id, match_index) if track_id != -1: # if candidate from prev frame matched, prevent it from matching another del bbox_list_prev_frame[match_index] del keypoints_list_prev_frame[match_index] # update current frame bbox bbox_det_dict = { "img_id": img_id, "det_id": det_id, "track_id": track_id, "bbox": bbox_det } bbox_dets_list.append(bbox_det_dict) # update current frame keypoints keypoints_dict = { "img_id": img_id, "det_id": det_id, "track_id": track_id, "keypoints": keypoints, 'kp_score': kp_score, 'bbox': bbox_det, 'proposal_score': proposal_score } keypoints_list.append(keypoints_dict) num_dets = len(bbox_dets_list) for det_id in range( num_dets): # detections for current frame bbox_det_dict = bbox_dets_list[det_id] keypoints_dict = keypoints_list[det_id] # assert (det_id == bbox_det_dict["det_id"]) # assert (det_id == keypoints_dict["det_id"]) if bbox_det_dict[ "track_id"] == -1: # this id means matching not found yet track_id = bbox_det_dict["track_id"] # track_id, match_index = get_track_id_SGCN(bbox_det_dict["bbox"], bbox_list_prev_frame, # keypoints_dict["keypoints"], # keypoints_list_prev_frame) if track_id != -1: # if candidate from prev frame matched, prevent it from matching another del bbox_list_prev_frame[match_index] del keypoints_list_prev_frame[match_index] bbox_det_dict["track_id"] = track_id keypoints_dict["track_id"] = track_id # if still can not find a match from previous frame, then assign a new id if track_id == -1 and not bbox_invalid( bbox_det_dict["bbox"]): bbox_det_dict["track_id"] = next_id keypoints_dict["track_id"] = next_id next_id += 1 # update frame bbox_dets_list_list.append(bbox_dets_list) keypoints_list_list.append(keypoints_list) # draw keypoints vis_frame(img, keypoints_list) # _pt1, _pt2 = _result['pt1'].numpy(), _result['pt2'].numpy() # pt1 = _pt1.astype(np.uint32) # pt2 = _pt2.astype(np.uint32) # for p1, p2 in zip(pt1, pt2): # cv2.rectangle(img, (p1[0], p1[1]), (p2[0], p2[1]), (34, 154, 11), 1) if CAR is not None: # No car detection car_track_id = 0 car_np = CAR new_car_bboxs = car_np[:, 0:4].astype(np.uint32) new_car_score = car_np[:, 4] car_dest_list = [] if img_id > 1: # First frame does not have previous frame car_bbox_list_prev_frame = car_dets_list_list[ img_id - 1].copy() else: car_bbox_list_prev_frame = [] # print('car bbox list prev frame ', len(car_bbox_list_prev_frame)) for c, score in zip(new_car_bboxs, new_car_score): car_bbox_det = c bbox_in_xywh = enlarge_bbox(car_bbox_det, enlarge_scale) bbox_det = x1y1x2y2_to_xywh(bbox_in_xywh) # obtain keypoints for each bbox position in the keyframe # print('img id ', img_id) if img_id == 0: # First frame, all ids are assigned automatically car_track_id = car_next_id car_next_id += 1 # print('if img id zero' , car_next_id) else: car_track_id, match_index = get_track_id_SpatialConsistency( bbox_det, car_bbox_list_prev_frame) # print(car_track_id, match_index) if car_track_id != -1: # if candidate from prev frame matched, prevent it from matching another del car_bbox_list_prev_frame[match_index] bbox_det_dict = { "img_id": img_id, "track_id": car_track_id, "bbox": bbox_det } car_dest_list.append(bbox_det_dict) # print() num_dets = len(car_dest_list) for det_id in range( num_dets): # detections for current frame car_bbox_det_dict = car_dest_list[det_id] # assert (det_id == bbox_det_dict["det_id"]) # assert (det_id == keypoints_dict["det_id"]) # print(Pose_matchercar_bbox_det_dict["track_id"]) if car_bbox_det_dict[ "track_id"] == -1: # this id means matching not found yet car_bbox_det_dict["track_id"] = car_next_id car_next_id += 1 # print('car net id ', car_next_id) self.tracking(car_dest_list, img_id) for car in car_dest_list: x, y, w, h = car['bbox'] track_id = car['track_id'] tracker = self.track_dict[track_id] history = tracker['history'] moved = np.sum(history[-10:]) last_moved = np.sum(history[-60:]) COLOR_MOVING = (0, 255, 0) COLOR_RED = (0, 0, 255) COLOR_INACTIVE = (255, 0, 0) cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_INACTIVE, 1) text_filled(img, (x, y), f'{track_id} Inactive', COLOR_INACTIVE) # if moved: # cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_MOVING, 1) # text_filled(img, (x, y), f'CAR {track_id} Active', COLOR_MOVING) # else: # # if last_moved: # cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_RED, 1) # text_filled(img, (x, y), f'CAR {track_id} Standstill', COLOR_RED) # # cropped = img[y:y+h, x:x+w,:] # filter = np.zeros(cropped.shape,dtype=img.dtype) # # print(cropped.shape, filter.shape) # filter[:,:,2] = 255 # # print(overlay.shape) # # cv2.rectangle(overlay, (0, 0), (w, h), COLOR_RED, -1) # overlayed = cv2.addWeighted(cropped,0.8,filter,0.2,0) # img[y:y+h, x:x+w,:] = overlayed[:,:,:] # else: # cv2.rectangle(img, (x, y), (x + w, y + h), COLOR_INACTIVE, 1) # text_filled(img, (x, y), f'{track_id} Inactive', COLOR_INACTIVE) car_dets_list_list.append(car_dest_list) else: car_dest_list = [] bbox_det_dict = { "img_id": img_id, "det_id": 0, "track_id": None, "bbox": [0, 0, 2, 2] } car_dest_list.append(bbox_det_dict) car_dets_list_list.append(car_dest_list) # if img_id != 0: # for car in car_dets_list_list[-1]: # car_track_id = car['track_id'] # if car_track_id is None: # continue # # car_bbox = car['bbox'] # for human in bbox_dets_list_list[-1]: # human_track_id = human['track_id'] # if human_track_id is None: # continue # hum_bbox = human['bbox'] # boxa = xywh_to_x1y1x2y2(hum_bbox) # boxb = xywh_to_x1y1x2y2(car_bbox) # x,y,w,h = x1y1x2y2_to_xywh(boxa) # area = iou(boxa,boxb) # # if area > 0.02: # cropped = img[y:y+h, x:x+w,:] # filter = np.zeros(cropped.shape,dtype=img.dtype) # filter[:,:,2] = 255 # overlayed = cv2.addWeighted(cropped,0.9,filter,0.1,0) # img[y:y+h, x:x+w,:] = overlayed[:,:,:] if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(1) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def detect_main(args, im_names, yolo_model, pose_net): # Load input images data_loader = ImageLoader(im_names, batchSize=args.detbatch, format='yolo').start() # Load detection loader det_loader = DetectionLoader(data_loader, model=yolo_model, batchSize=args.detbatch).start() det_processor = DetectionProcessor(det_loader).start() runtime_profile = { 'dt': [], 'pt': [], 'pn': [] } # Init data writer # writer = DataWriter(args.save_video).start() data_len = data_loader.length() fall_res_all = [] batchSize = args.posebatch for i in range(data_len): start_time = getTime() with torch.no_grad(): (inps, orig_img, im_name, boxes, scores, pt1, pt2) = det_processor.read() if boxes is None or boxes.nelement() == 0: # writer.save(None, None, None, None, None, orig_img, im_name.split('/')[-1]) continue ckpt_time, det_time = getTime(start_time) runtime_profile['dt'].append(det_time) # Pose Estimation # print(im_name) datalen = inps.size(0) leftover = 0 if (datalen) % batchSize: leftover = 1 num_batches = datalen // batchSize + leftover hm = [] for j in range(num_batches): inps_j = inps[j * batchSize:min((j + 1) * batchSize, datalen)].cuda() hm_j = pose_net(inps_j) hm.append(hm_j) hm = torch.cat(hm) ckpt_time, pose_time = getTime(ckpt_time) runtime_profile['pt'].append(pose_time) hm = hm.cpu() # writer.save(boxes, scores, hm, pt1, pt2, orig_img, im_name.split('/')[-1]) fall_res = [] fall_res.append(im_name.split('/')[-1]) if boxes is None: cv2.imwrite(opt.outputpath + '/' + im_name.split('/')[-1], img) else: if opt.matching: preds = getMultiPeakPrediction( hm, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction(hm, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'imgname': im_name, 'result': result} img = vis_frame(orig_img, result) for human in result['result']: keypoint = human['keypoints'] keypoint = keypoint.numpy() xmax = max(keypoint[:, 0]) xmin = min(keypoint[:, 0]) ymax = max(keypoint[:, 1]) ymin = min(keypoint[:, 1]) w = xmax - xmin h = ymax - ymin distance = abs((keypoint[15][1] + keypoint[16][1]) / 2 - (keypoint[11][1] + keypoint[12][1]) / 2) if w / h >= 0.95: cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2) font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(img, 'Warning!Fall', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 0, 255), 2) fall_res.append([xmin,ymin,xmax,ymax]) ''' print('1 location:[%f,' % (xmin) + '%f]' % (ymin) + ' [%f,' % (xmax) + '%f]' % ( ymin) + ' [%f,' % ( xmin) + '%f]' % (ymax) + ' [%f,' % (xmax) + '%f]' % (ymax)) ''' else: if distance < 55: cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2) font = cv2.FONT_HERSHEY_SIMPLEX cv2.putText(img, 'Warning!Fall!', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 255, 0), 2) fall_res.append(1) fall_res.append([xmin,ymin,xmax,ymax]) ''' print('1 location:[%f,' % (xmin) + '%f]' % (ymin) + ' [%f,' % (xmax) + '%f]' % ( ymin) + ' [%f,' % ( xmin) + '%f]' % (ymax) + ' [%f,' % (xmax) + '%f]' % (ymax)) ''' else: cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2) #cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) print(fall_res) cv2.imwrite(opt.outputpath + '/' + im_name.split('/')[-1], img) ckpt_time, post_time = getTime(ckpt_time) runtime_profile['pn'].append(post_time) fall_res_all.append(fall_res) return fall_res_all
def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) # print('++++++++++++++++++++we will print original image+++++++++++++++++++++++++') # cv2.imwrite(os.path.join(opt.outputpath, 'det_img', im_name), orig_img) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) if opt.matching: preds = getMultiPeakPrediction(hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) pose_result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'imgname': im_name, 'result': pose_result} # occlusion evalution result = occlud_eval(result) '''pan edit on 20200406''' # bbox semantic code result = bbox_code_process(result) self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
coords = [] result = [] for i in range(num_kps): heat = heatmap[:, :, i] heat = gaussian_filter(heat, sigma=5) ind = pose_processing(heat, max_locs[i]) coord_x = int((ind[1] + 1) / scale_w) coord_y = int((ind[0] + 1) / scale_h) coords.append((coord_x, coord_y)) result.append({'keypoints': coords, 'kp_score': scores}) imgname = str(frame_index) + '.jpg' result = {'imgname': imgname, 'result': result} img = vis_frame(img_data, result) if show_txt >= 0: img = cv2.putText(img, 'ONLY ONE PERSON IS ALLOWED!', (40, 40), fontFace=cv2.FONT_ITALIC, fontScale=1, color=(0, 0, 255), thickness=3) show_txt -= 1 else: # print warning img_txt = cv2.putText(img_data, 'ONLY ONE PERSON IS ALLOWED!', (40, 40), fontFace=cv2.FONT_ITALIC, fontScale=1,
def detect_main(im_name, orig_img, det_model, pose_model, opt): args = opt mode = args.mode inp_dim = int(opt.inp_dim) dim = orig_img.shape[1], orig_img.shape[0] img_ = (letterbox_image(orig_img, (inp_dim, inp_dim))) img_ = img_[:, :, ::-1].transpose((2, 0, 1)).copy() img = torch.from_numpy(img_).float().div(255.0).unsqueeze(0) img = [img] orig_img = [orig_img] im_name = [im_name] im_dim_list = [dim] # img.append(img_k) # orig_img.append(orig_img_k) # im_name.append(im_name_k) # im_dim_list.append(im_dim_list_k) with torch.no_grad(): # Human Detection img = torch.cat(img) im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2) # im_dim_list_ = im_dim_list # DetectionLoader det_inp_dim = int(det_model.net_info['height']) assert det_inp_dim % 32 == 0 assert det_inp_dim > 32 # res_n = 0 with torch.no_grad(): img = img.cuda() prediction = det_model(img, CUDA=True) # a tensor boxes_chair = get_box(prediction, det_inp_dim, im_dim_list, opt.confidence, opt.num_classes, 56) boxes_sofa = get_box(prediction, det_inp_dim, im_dim_list, opt.confidence, opt.num_classes, 57) boxes_bed = get_box(prediction, det_inp_dim, im_dim_list, opt.confidence, opt.num_classes, 59) dets = dynamic_write_results(prediction, opt.confidence, opt.num_classes, 0, nms=True, nms_conf=opt.nms_thesh) if isinstance(dets, int) or dets.shape[0] == 0: # cv2.imwrite('err_result/no_person/'+im_name[0][0:-4]+'_re.jpg', orig_img[0]) return [] dets = dets.cpu() im_dim_list = torch.index_select(im_dim_list, 0, dets[:, 0].long()) scaling_factor = torch.min(det_inp_dim / im_dim_list, 1)[0].view(-1, 1) # coordinate transfer dets[:, [1, 3]] -= (det_inp_dim - scaling_factor * im_dim_list[:, 0].view(-1, 1)) / 2 dets[:, [2, 4]] -= (det_inp_dim - scaling_factor * im_dim_list[:, 1].view(-1, 1)) / 2 dets[:, 1:5] /= scaling_factor for j in range(dets.shape[0]): dets[j, [1, 3]] = torch.clamp(dets[j, [1, 3]], 0.0, im_dim_list[j, 0]) dets[j, [2, 4]] = torch.clamp(dets[j, [2, 4]], 0.0, im_dim_list[j, 1]) boxes = dets[:, 1:5] scores = dets[:, 5:6] boxes_k = boxes[dets[:, 0] == 0] if isinstance(boxes_k, int) or boxes_k.shape[0] == 0: boxes = None scores = None inps = None pt1 = None pt2 = None else: inps = torch.zeros(boxes_k.size(0), 3, opt.inputResH, opt.inputResW) pt1 = torch.zeros(boxes_k.size(0), 2) pt2 = torch.zeros(boxes_k.size(0), 2) orig_img = orig_img[0] im_name = im_name[0] boxes = boxes_k scores = scores[dets[:, 0] == 0] # orig_img[k], im_name[k], boxes_k, scores[dets[:, 0] == k], inps, pt1, pt2 # DetectionProcess with torch.no_grad(): if boxes is None or boxes.nelement() == 0: pass else: inp = im_to_torch(cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB)) inps, pt1, pt2 = crop_from_dets(inp, boxes, inps, pt1, pt2) # self.Q.put((inps, orig_img, im_name, boxes, scores, pt1, pt2)) batchSize = args.posebatch # fall_res_all = [] for i in range(1): with torch.no_grad(): if boxes is None or boxes.nelement() == 0: # writer.save(None, None, None, None, None, orig_img, im_name.split('/')[-1]) # res_n = 0 continue # Pose Estimation datalen = inps.size(0) leftover = 0 if (datalen) % batchSize: leftover = 1 num_batches = datalen // batchSize + leftover hm = [] for j in range(num_batches): inps_j = inps[j * batchSize:min((j + 1) * batchSize, datalen)].cuda() hm_j = pose_model(inps_j) hm.append(hm_j) hm = torch.cat(hm) hm = hm.cpu() # writer.save(boxes, scores, hm, pt1, pt2, orig_img, im_name.split('/')[-1]) fall_res = [] keypoint_res = [] # fall_res.append(im_name.split('/')[-1]) if opt.matching: preds = getMultiPeakPrediction(hm, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction( hm, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {'imgname': im_name, 'result': result} # img = orig_img img = vis_frame(orig_img, result) for human in result['result']: keypoint = human['keypoints'] kp_scores = human['kp_score'] keypoint = keypoint.numpy() xmax = max(keypoint[:, 0]) xmin = min(keypoint[:, 0]) ymax = max(keypoint[:, 1]) ymin = min(keypoint[:, 1]) box_hm = [xmin, ymin, xmax, ymax] kp_num = 0 for i in range(len(kp_scores)): if kp_scores[i] > 0.05: kp_num += 1 if kp_num < 10: # cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2) fall_res.append([False, xmin, ymin, xmax, ymax]) # print("kp_num:"+str(kp_num)) continue overlap = [] for box in boxes_chair: overlap.append(compute_overlap(box_hm, box)) for box in boxes_sofa: overlap.append(compute_overlap(box_hm, box)) for box in boxes_bed: overlap.append(compute_overlap(box_hm, box)) if len(overlap) > 0 and max(overlap) >= 0.6: # res_n = 0 fall_res.append([False, xmin, ymin, xmax, ymax]) # cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2) # print("overlap:"+str(overlap)) continue w = xmax - xmin h = ymax - ymin ratio = w / h # distance = abs((keypoint[15][1] + keypoint[16][1]) / 2 - (keypoint[11][1] + keypoint[12][1]) / 2) xhead = (keypoint[1][0] + keypoint[2][0] + keypoint[2][0] + keypoint[3][0] + keypoint[4][0]) / 4 yhead = (keypoint[1][1] + keypoint[2][1] + keypoint[2][1] + keypoint[3][1] + keypoint[4][1]) / 4 xfeet = (keypoint[15][0] + keypoint[16][0]) / 2 yfeet = (keypoint[15][1] + keypoint[16][1]) / 2 d_ear = (abs(keypoint[3][0] - keypoint[4][0])**2 + abs(keypoint[3][1] - keypoint[4][1])**2)**0.5 r = (w**2 + h**2)**0.5 / d_ear if kp_scores[3] > 0.05 and kp_scores[4] > 0.05 and r < 4: fall_res.append([False, xmin, ymin, xmax, ymax]) # cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2) # print("r<4") continue # distance = abs((keypoint[15][1] + keypoint[16][1]) / 2 - (keypoint[11][1] + keypoint[12][1]) / 2) # xhead_foot = abs(xfeet - xhead) # yhead_foot = abs(yfeet - yhead) # dhead_foot = (xhead_foot ** 2 + yhead_foot ** 2) ** 0.5 # ratio = yhead_foot / dhead_foot if min(kp_scores[3], kp_scores[4], kp_scores[15], kp_scores[16]) > 0.05 and yfeet < (keypoint[3][1] + keypoint[4][1]) / 2: # cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2) # font = cv2.FONT_HERSHEY_SIMPLEX # cv2.putText(img, 'Warning!Fall!', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 255, 0), 2) fall_res.append([True, xmin, ymin, xmax, ymax]) keypoint_res.append(keypoint) # res_n = 2 elif w / h >= 1.0: # cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (0, 0, 255), 2) # font = cv2.FONT_HERSHEY_SIMPLEX # cv2.putText(img, 'Warning!Fall', (int(xmin + 10), int(ymax - 10)), font, 1, (0, 0, 255), 2) fall_res.append([True, xmin, ymin, xmax, ymax]) keypoint_res.append(keypoint) # res_n = 1 else: # cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 0, 0), 2) # print("normal") fall_res.append([False, xmin, ymin, xmax, ymax]) # res_n = 0 # cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) ''' for box in boxes_chair: cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (255, 255, 0), 2) for box in boxes_sofa: cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (0, 255, 255), 2) for box in boxes_bed: cv2.rectangle(img, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), (255, 0, 255), 2) cv2.imwrite('err_result/false/'+im_name[0:-4]+'_re.jpg', img) ''' return keypoint_res
def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite( os.path.join(self.outputpath, self.dir_folder, im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) # if opt.matching: # preds = getMultiPeakPrediction( # hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) # result = matching(boxes, scores.numpy(), preds) # else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) # (hm_data, pt1, pt2, 320, 256, 80, 64) result = pose_nms(boxes, scores, preds_img, preds_scores) # (bbox定位list, bbox评分list, 位姿定位list, 位姿评分list) # result[ {'keypoints', 'kp_score', 'proposal_score'}, {---}, ...], pPose_nms.py line-114 result = {'imgname': im_name, 'result': result} # self.final_result.append(result) self.show_img = vis_frame(orig_img, result) cv2.imwrite( os.path.join(self.outputpath, self.dir_folder, im_name), img) # if opt.save_img or opt.save_video or opt.vis: # img = vis_frame(orig_img, result) # if opt.vis: # cv2.imshow("AlphaPose Demo", img) # cv2.waitKey(30) # if opt.save_img: # cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) # if opt.save_video: # self.stream.write(img) else: # time.sleep(0.1) pass
def update(self): # keep looping infinitely while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite(os.path.join(opt.outputpath, "vis", f"{im_name}.jpg"), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) if opt.matching: preds = getMultiPeakPrediction( hm_data, pt1.numpy(), pt2.numpy(), opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW, ) result = matching(boxes, scores.numpy(), preds) else: preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW, ) result = pose_nms(boxes, scores, preds_img, preds_scores) result = {"imgname": im_name, "result": result} self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis or self.result_handler: # Draw poses on frame or exececute custom result_handler if self.result_handler is None: img = vis_frame(orig_img, result) else: img = self.result_handler(orig_img, result) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite(os.path.join(opt.outputpath, "vis", f"{im_name}.jpg"), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)
def update(self): # keep looping infinitely temp_kps=[] while True: # if the thread indicator variable is set, stop the # thread if self.stopped: if self.save_video: self.stream.release() return # otherwise, ensure the queue is not empty if not self.Q.empty(): (boxes, scores, hm_data, pt1, pt2, orig_img, im_name) = self.Q.get() orig_img = np.array(orig_img, dtype=np.uint8) if boxes is None: if opt.save_img or opt.save_video or opt.vis: img = orig_img if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: # location prediction (n, kp, 2) | score prediction (n, kp, 1) preds_hm, preds_img, preds_scores = getPrediction( hm_data, pt1, pt2, opt.inputResH, opt.inputResW, opt.outputResH, opt.outputResW) result = pose_nms(boxes, scores, preds_img, preds_scores) pos = result[0]['keypoints'].unsqueeze(0).numpy() pos = self.aligner.align_points(pos)[0] pos = (pos[..., :2] - 129) / 255 pos = torch.FloatTensor(pos) kp = torch.cat((pos, result[0]['kp_score']), 1) kp = kp.unsqueeze(0) if len(temp_kps) < 9: kp = kp.reshape([1, -1]).cuda() temp_kps.append(kp) kp = kp.repeat(9, 1).reshape(1, -1) outputs = self.pos_reg_model(kp) _, preds = torch.max(outputs, 1) classidx = preds.cpu() result[0]['class'] = str(float(classidx)) else: kp = kp.cuda().reshape(1,-1) temp_kps.append(kp) temp_kps.pop(0) _temp_kps = torch.cat(temp_kps) _temp_kps.cuda() _temp_kps = _temp_kps.reshape([1, -1]) outputs = self.pos_reg_model(_temp_kps) _, preds = torch.max(outputs, 1) classidx = preds.cpu() result[0]['class'] = str(float(classidx)) # print(preds) result = { 'imgname': im_name, 'result': result } self.result_Q.put((boxes, classidx)) self.final_result.append(result) if opt.save_img or opt.save_video or opt.vis: img = vis_frame(orig_img, result) if opt.vis: cv2.imshow("AlphaPose Demo", img) cv2.waitKey(30) if opt.save_img: cv2.imwrite(os.path.join(opt.outputpath, 'vis', im_name), img) if opt.save_video: self.stream.write(img) else: time.sleep(0.1)