def index(): pnet, rnet, onet = create_mtcnn_net( p_model_path="./original_model/pnet_epoch.pt", r_model_path="./original_model/rnet_epoch.pt", o_model_path="./original_model/onet_epoch.pt", use_cuda=False) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24) #get_json=flask.request.get_json(force=True) print(request.data) base_data = request.json['image'] img = base64_to_image(base_data) bboxs, landmarks = mtcnn_detector.detect_face(img) #初始化一下json res = {} faces = {} if bboxs.shape[0] < 1: res["success"] = False res["faces_detected"] = faces return flask.jsonify(res) else: res["success"] = True #这里开始处理一幅图中有多个人脸的情况 for i in range(bboxs.shape[0]): x1 = int(bboxs[i][0]) x2 = int(bboxs[i][2]) y1 = int(bboxs[i][1]) y2 = int(bboxs[i][3]) face = img[y1:y2, x1:x2] face_name = "face_" + str(i) return_base64 = image_to_base64(face) faces[face_name] = return_base64 res["faces_detected"] = faces return flask.jsonify(res)
def test(inoutDir, outputDir, model): # 原模型的P,R,net + 自行训练后的Onet,展示并保存检测后的图片 pnet, rnet, onet_jiang = create_mtcnn_net( p_model_path="./original_model/pnet_epoch.pt", r_model_path="./original_model/rnet_epoch.pt", o_model_path="./original_model/" + model + ".pt", use_cuda=False) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet_jiang, min_face_size=24) files = os.listdir(inoutDir) i = 0 for image in files: i += 1 image = os.path.join("./lfpw_test/", image) img = cv2.imread(image) img_bg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) bboxs, landmarks1 = mtcnn_detector.detect_face(img) # 原始的图片用原始网络检测 vis_face(img_bg, bboxs, landmarks1, outputDir + model + "-" + str(i) + ".jpg") # 保存图片
def detect(self): pnet, rnet, onet = create_mtcnn_net(p_model_path=self.p_net_m, r_model_path=self.r_net_m, o_model_path=self.o_net_m, use_cuda=True) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24, threshold=[0.1, 0.1, 0.1]) event_list = os.listdir(self.image_dir) for event in event_list: print(event) event_dir = os.path.join(self.image_dir, event) res_dir = os.path.join(self.result_dir, event) if not os.path.exists(res_dir): os.makedirs(res_dir) images_list = os.listdir(event_dir) for images in images_list: images_path = os.path.join(event_dir, images) img = cv2.imread(images_path) bboxs, landmarks = mtcnn_detector.detect_face(img) if bboxs.shape[0] != 0: bboxs[:, 2] = bboxs[:, 2] - bboxs[:, 0] bboxs[:, 3] = bboxs[:, 3] - bboxs[:, 1] bboxs[:, :4] = np.round(bboxs[:, :4]) """ print(bboxs) save_name = 'r_304.jpg' vis_face(img,bboxs,landmarks, save_name) """ fpath = os.path.join(res_dir, images[:-4] + '.txt') f = open(fpath, 'w') f.write(images[:-4] + '\n') f.write(str(bboxs.shape[0]) + '\n') for i in range(bboxs.shape[0]): f.write('{:.0f} {:.0f} {:.0f} {:.0f} {:.3f}\n'.format( bboxs[i, 0], bboxs[i, 1], bboxs[i, 2], bboxs[i, 3], bboxs[i, 4])) f.close()
net.cuda() cap = cv2.VideoCapture(0) cap.set(3,640) cap.set(4,640) success, frame = cap.read() #frame = cv2.imread('images/1.png') #frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) while True: if success: success = 0 #t = time.time() # *********************Face Detection********************** bboxs, landmarks = mtcnn_detector.detect_face(frame) #t1 = time.time() - t #fps1 = 1 / t1 for i in range(bboxs.shape[0]): bbox = bboxs[i, :4] x1 = int(bbox[0]) y1 = int(bbox[1]) x2 = int(bbox[2]) y2 = int(bbox[3]) roi = frame[y1:y2, x1:x2] roi = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) roi = cv2.resize(roi, (49, 49)) roi = roi[:, :, np.newaxis] roi = np.concatenate((roi, roi, roi), axis=2)
import cv2 from mtcnn.core.detect import create_mtcnn_net, MtcnnDetector from mtcnn.core.vision import vis_face if __name__ == '__main__': pnet, rnet, onet = create_mtcnn_net( p_model_path="./original_model/pnet_epoch.pt", r_model_path="./original_model/rnet_epoch.pt", o_model_path="./original_model/onet_epoch.pt", use_cuda=False) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24) img = cv2.imread("./s_l.jpg") img_bg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # b, g, r = cv2.split(img) # img2 = cv2.merge([r, g, b]) bboxs, landmarks = mtcnn_detector.detect_face(img) # print box_align save_name = 'r_4.jpg' vis_face(img_bg, bboxs, landmarks, save_name)
# parser.add_argument('--onet_file', type=str, default='model/onet.pt') # parser.add_argument('--onet_file', type=str, default='model/onet_v1/onet_epoch_7.pt') # parser.add_argument('--onet_file', type=str, default='model/onet_v2/onet_epoch_6.pt') parser.add_argument('--onet_file', type=str, default='model/onet_v3/onet_epoch_10.pt') parser.add_argument('--use_cuda', type=bool, default=True) # TODO parser.add_argument('--gpu_ids', type=list, default=[0, 1]) # TODO parser.add_argument('--prob_thres', type=list, default=[0.6, 0.7, 0.7]) args = parser.parse_args() return args if __name__ == '__main__': imglists = [s.split('.')[0] for s in os.listdir('aku_imgs/')] mtcnn_detector = MtcnnDetector(parse_args()) for img_name in imglists: try: img = cv2.imread('aku_imgs/%s.jpg' % img_name) bboxs, landmarks = mtcnn_detector.detect_face(img, verbose=False) save_name = 'result/r3_%s.jpg' % img_name print('save img name : %s' % save_name) visual_face(img, bboxs, landmarks, save_name) except Exception as e: print(e)
class HistOccBlock(object): def __init__(self, args): self.args = args self.imgs_list = None self.model = MtcnnDetector(self.args) self.pdf_list = None def _prepare_folder(self): ''' Prepare the face-model, imgs_list ''' # imgs_list folder_path = os.path.join(self.args.data_dir, self.args.folder) imgs_list = [] for img_name in os.listdir(folder_path): # idx = folder.split('_')[-1] # img_name = '%s_%d.jpg' % (self.args.img_type, int(idx)) imgs_list.append(os.path.join(folder_path, img_name)) self.imgs_list = imgs_list print('there are %d imgs in %s' % (len(imgs_list), self.args.folder)) return imgs_list def _prepare_csv(self): ''' Prepare the face-model, imgs_list ''' csv_file = os.path.join(self.args.data_dir, 'csv_raw', self.args.csv_file) df_csv = pd.read_csv(csv_file) print('csv_file.shape : ', df_csv.shape) df_test = None if self.args.check_mode == 'pos': check_mode = -1 elif self.args.check_mode == 'neg': check_mode = 0 else: check_mode = 1 print('attention, evaluate-mode was started ...') if check_mode < 1: df_csv = df_csv[df_csv['anno_label'] == check_mode] imgs_list = [] for idx, row in df_csv.iterrows(): img_name = '/'.join(row['img_path'].split('/')[-2:]) img_path = os.path.join(self.args.data_dir, img_name) imgs_list.append(img_path) self.imgs_list = imgs_list else: df_test = [] for idx, row in df_csv.iterrows(): img_name = '/'.join(row['img_path'].split('/')[-2:]) img_path = os.path.join(self.args.data_dir, img_name) df_test.append([img_path, row['img_type'], row['anno_label']]) df_test = pd.DataFrame(df_test, columns=df_csv.columns) print('after filtering, df_test.shape : ', df_test.shape) print('imgs_list was prepared ...') return df_test def _fliter_doc_bbox(self, bboxes, landmarks): ''' Filter the face_box on card ''' area = (bboxes[:, 2] - bboxes[:, 0] + 1) * (bboxes[:, 3] - bboxes[:, 1] + 1) * -1 area_index = area.argsort() bbox = bboxes[area_index[0]] landmark = landmarks[area_index[0]] #prob_index = (bboxes[-1]*-1).argsort() # assist # if bboxes.shape[0] == 2 or area_index[0] == prob_index[0]: # bbox = bboxes[area_index[0]] # landmark = landmark[area_index[0]] return bbox, landmark def _fetch_block(self, img, bbox, landmark, lmk_flag = True): ''' Crop the chin_block of the detected face ''' landmark = landmark.reshape(-1,2) if lmk_flag: left_down = landmark[6] nose_point = landmark[33] # point-34 | nose right_down = landmark[10] chin_point = landmark[8] # point-9 | chin x1, y1 = int(left_down[0]), int(nose_point[1]) x2, y2 = int(right_down[0]), int(chin_point[1]) else: x1, y1 = int(bbox[0]), int(bbox[1]) x2, y2 = int(bbox[2]), int(bbox[3]) crop_block = img[y1:y2 + 1, x1:x2 + 1, :] return crop_block def _statistics(self, block): ''' Statistics the block-pixels info ''' height, width, _ = block.shape num_pixels = height * width bgr_prob = np.zeros((3, 256), dtype=np.int) for y in range(height): for x in range(width): pixel = block[y,x] bgr_prob[0, pixel[0]] += 1 bgr_prob[1, pixel[1]] += 1 bgr_prob[2, pixel[2]] += 1 bgr_prob = bgr_prob / num_pixels return bgr_prob def _hist_go(self): ''' Statistic the info of resz_block step - 1. detect face with trained model step - 2. filter the doc_bbox step - 3. crop and resize the target block step - 4. get the statistics_info ''' pdf_list = [] for img_path in self.imgs_list: try: img = cv2.imread(img_path) bboxes, landmarks = self.model.detect_face(img, verbose=False) # save_name = 'result/r_%s' % img_path.split('/')[-1] # vis_utils.visual_face(img, bboxes, landmarks, save_name) print((img_path, bboxes.shape)) except Exception as e: print(e) else: if bboxes.shape[0] == 0: print('No face detected in %s' % img_path) continue else: bbox, landmark = self._fliter_doc_bbox(bboxes, landmarks) block = self._fetch_block(img, bbox, landmark) bgr_prob = self._statistics(block) pdf_list.append(bgr_prob.reshape(1, -1)[0]) self.pdf_list = pdf_list date_stamp = self.args.csv_file.split('_')[-1].split('.')[0] save_name = 'pdf/npy_data/%s_details_%s.npy' % (self.args.check_mode, date_stamp) print('npy_data was savd in %s' % save_name) np.save(save_name, pdf_list) print('hist-module was finished ...') def runner(self, vis = False): ''' Pipeline of HistOccBlock ''' self._prepare_csv() self._hist_go()
def show_with_camera(): eye_class_dict = {0: "open_eye", 1: "close_eye", 2: "other"} point_nums = 24 threshold = [0.6, 0.7, 0.7] data_trans = Transforms.Compose([ Transforms.Resize((24, 24)), Transforms.ToTensor(), Transforms.Normalize((0.45, 0.448, 0.455), (0.082, 0.082, 0.082)), # Transforms.Normalize((0.407, 0.405, 0.412), (0.087, 0.087, 0.087)), ]) mixnet = MixNet(input_size=(24, 24), num_classes=3) # eye_class_dict = {0:"open_eye",1:"close_eye"} # weight_dict = torch.load("weight/signal_eye/Mixnet_epoch_29.pth") weight_dict = torch.load( "/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/weight/mix_mbhk_change_signal_eye_24_24/Mixnet_epoch_59.pth" ) new_state_dict = OrderedDict() for k, v in weight_dict.items(): name = k[7:] new_state_dict[name] = v mixnet.load_state_dict(new_state_dict) # stat(net,(3,48,48)) mixnet.to('cuda:0') mixnet.eval() pnet, rnet, onet = create_mtcnn_net( p_model_path=r'model_store/final/pnet_epoch_19.pt', r_model_path=r'model_store/final/rnet_epoch_7.pt', o_model_path=r'model_store/final/onet_epoch_92.pt', use_cuda=True) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24, threshold=threshold) videos_root_path = 'test_video/20200506143954001_0.avi' save_path_root = 'result_video/camera_test_20210301.avi' cap = cv2.VideoCapture(0) fourcc = cv2.VideoWriter_fourcc(*'XVID') fps = cap.get(cv2.CAP_PROP_FPS) size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))) # tpa # fname = os.path.splitext(os.path.split(tpa)[1])[0] # save_path = os.path.join("/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/result_video/data(2)",fname+".avi") out = cv2.VideoWriter(save_path_root, fourcc, fps, size) while True: ret, frame = cap.read() if ret: copy_frame = frame.copy() left_right_eye = [] bboxs, landmarks, wearmask = mtcnn_detector.detect_face(frame, rgb=True) if landmarks is not None: for i in range(landmarks.shape[0]): landmarks_one = landmarks[i, :] landmarks_one = landmarks_one.reshape((point_nums, 2)) left_eye = np.array(landmarks_one[[6, 8, 10, 11, 14], :]) xmin = np.min(left_eye[:, 0]) ymin = np.min(left_eye[:, 1]) xmax = np.max(left_eye[:, 0]) ymax = np.max(left_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) right_eye = np.array(landmarks_one[[7, 9, 12, 13, 15], :]) xmin = np.min(right_eye[:, 0]) ymin = np.min(right_eye[:, 1]) xmax = np.max(right_eye[:, 0]) ymax = np.max(right_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) for j in [*left_eye, *right_eye]: cv2.circle(frame, (int(j[0]), int(j[1])), 2, (255, 0, 0), -1) crop_img = [] for xmin, ymin, xmax, ymax in left_right_eye: w, h = xmax - xmin, ymax - ymin # 随机扩展大小0.05-0.15 k = 0.1 ratio = h / w if ratio > 1: ratio = ratio - 1 xmin -= (ratio / 2 * w + k * h) ymin -= (k * h) xmax += (ratio / 2 * w + k * h) ymax += (k * h) else: ratio = w / h - 1 xmin -= (k * w) ymin -= (ratio / 2 * h + k * w) xmax += (k * w) ymax += (ratio / 2 * h + k * w) cv2.rectangle(frame, (int(xmin), int(ymin)), (int(xmax), int(ymax)), (0, 255, 255), 2) temp_img = copy_frame[int(ymin):int(ymax), int(xmin):int(xmax)] # temp_img = cv2.resize(temp_img,(24,24)) crop_img.append(temp_img) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR) if len(crop_img) < 2: cv2.imshow("test", frame) tget_in = cv2.waitKey(10) # print(ord('q'),tget_in) if tget_in == ord('q'): print("get out") break out.write(frame) continue # compose_img = np.hstack((crop_img[0],crop_img[1])) t_result = [] for i in crop_img: i = cv2.cvtColor(i, cv2.COLOR_BGR2GRAY) i = cv2.cvtColor(i, cv2.COLOR_GRAY2RGB) compose_img = Image.fromarray(i) img = data_trans(compose_img) img = img.unsqueeze(0) with torch.no_grad(): outputs = mixnet(img.to('cuda:0')) result = torch.max(outputs, 1)[1] t_result.append(result.item()) if 0 in t_result: eye_class = "open_eye" cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(255,0,255) \ ,thickness=2) elif 1 in t_result: eye_class = "close_eye" cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(0,255,0) \ ,thickness=2) else: eye_class = "other" cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(0,0,255) \ ,thickness=2) cv2.imshow("test", frame) tget_in = cv2.waitKey(10) if tget_in == ord('q'): print("get out") break # eye_class = "open_eye" if 0 in t_result else "close_eye" # cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(255,0,255) \ # if 0 in t_result else (255,255,0),thickness=2) out.write(frame) else: print("finish") break
def dete_picture(): eye_class_dict = {0: "open_eye", 1: "close_eye", 2: "other"} point_nums = 24 threshold = [0.6, 0.7, 0.7] data_trans = Transforms.Compose([ Transforms.Resize((24, 24)), Transforms.ToTensor(), Transforms.Normalize((0.45, 0.448, 0.455), (0.082, 0.082, 0.082)), # Transforms.Normalize((0.407, 0.405, 0.412), (0.087, 0.087, 0.087)), ]) mixnet = MixNet(input_size=(24, 24), num_classes=3) # eye_class_dict = {0:"open_eye",1:"close_eye"} # weight_dict = torch.load("weight/signal_eye/Mixnet_epoch_29.pth") weight_dict = torch.load( "/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/weight/relabel_mix_24_24_20210302/Mixnet_epoch_59.pth" ) new_state_dict = OrderedDict() for k, v in weight_dict.items(): name = k[7:] new_state_dict[name] = v mixnet.load_state_dict(new_state_dict) # stat(net,(3,48,48)) mixnet.to('cuda:0') mixnet.eval() pnet, rnet, onet = create_mtcnn_net( p_model_path=r'model_store/final/pnet_epoch_19.pt', r_model_path=r'model_store/final/rnet_epoch_7.pt', o_model_path=r'model_store/final/onet_epoch_92.pt', use_cuda=True) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24, threshold=threshold) img_file = "/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/test_video/caiji_0123" img_save = "/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/result_video/relabel_img_result_adma_01" img_path = [ os.path.join(img_file, file_name) for file_name in glob.glob(os.path.join(img_file, "*.jpg")) ] # videos_root_path = 'test_video/DMS_RAW_Nebula_20201201-143038_518.mp4' # save_path_root = 'result_video/24_24_DMS_RAW_Nebula_20201201-143038_518.avi' # cap = cv2.VideoCapture(videos_root_path) # fourcc = cv2.VideoWriter_fourcc(*'XVID') # fps = cap.get(cv2.CAP_PROP_FPS) # size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))) # tpa # fname = os.path.splitext(os.path.split(tpa)[1])[0] # save_path = os.path.join("/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/result_video/data(2)",fname+".avi") # out = cv2.VideoWriter(save_path_root, fourcc, fps, size) for img_p in tqdm(img_path): frame = cv2.imread(img_p) copy_frame = frame.copy() left_right_eye = [] bboxs, landmarks, wearmask = mtcnn_detector.detect_face(frame, rgb=True) if landmarks is not None: for i in range(landmarks.shape[0]): landmarks_one = landmarks[i, :] landmarks_one = landmarks_one.reshape((point_nums, 2)) left_eye = np.array(landmarks_one[[6, 8, 10, 11, 14], :]) xmin = np.min(left_eye[:, 0]) ymin = np.min(left_eye[:, 1]) xmax = np.max(left_eye[:, 0]) ymax = np.max(left_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) right_eye = np.array(landmarks_one[[7, 9, 12, 13, 15], :]) xmin = np.min(right_eye[:, 0]) ymin = np.min(right_eye[:, 1]) xmax = np.max(right_eye[:, 0]) ymax = np.max(right_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) for j in [*left_eye, *right_eye]: cv2.circle(frame, (int(j[0]), int(j[1])), 2, (255, 0, 0), -1) crop_img = [] for xmin, ymin, xmax, ymax in left_right_eye: w, h = xmax - xmin, ymax - ymin # 随机扩展大小0.05-0.15 k = 0.1 ratio = h / w if ratio > 1: ratio = ratio - 1 xmin -= (ratio / 2 * w + k * h) ymin -= (k * h) xmax += (ratio / 2 * w + k * h) ymax += (k * h) else: ratio = w / h - 1 xmin -= (k * w) ymin -= (ratio / 2 * h + k * w) xmax += (k * w) ymax += (ratio / 2 * h + k * w) cv2.rectangle(frame, (int(xmin), int(ymin)), (int(xmax), int(ymax)), (0, 255, 255), 2) temp_img = copy_frame[int(ymin):int(ymax), int(xmin):int(xmax)] # temp_img = cv2.resize(temp_img,(24,24)) crop_img.append(temp_img) if len(crop_img) < 2: img_name = os.path.split(img_p)[-1] cv2.imwrite(os.path.join(img_save, img_name), frame) # out.write(frame) continue # compose_img = np.hstack((crop_img[0],crop_img[1])) result_buff = [] score_buff = [] for i in crop_img: i = cv2.cvtColor(i, cv2.COLOR_BGR2RGB) compose_img = Image.fromarray(i) img = data_trans(compose_img) img = img.unsqueeze(0) with torch.no_grad(): outputs = mixnet(img.to('cuda:0')) spft_max = torch.nn.functional.softmax(outputs, dim=1) score_buff.append(spft_max.cpu().numpy()) # 0,1->data,id score, result = torch.max(spft_max, 1) result_buff.append([result.item(), score]) bias = 30 eye_bias = 100 for i in range(2): t_result = result_buff[i][0] if 0 == t_result: # eye_class = "close_eye" # cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(0,255,0) \ # ,thickness=2) eye_class = "open_eye:{:.2f}".format( result_buff[i][1].cpu().item()) cv2.putText(frame,eye_class,(int(left_right_eye[i][0])-eye_bias, int(left_right_eye[i][1])-bias),cv2.FONT_HERSHEY_COMPLEX,0.6,(255,0,255) \ ,thickness=2) elif 1 == t_result: # eye_class = "open_eye" # cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(255,0,255) \ # ,thickness=2) eye_class = "close_eye:{:.2f}".format( result_buff[i][1].cpu().item()) cv2.putText(frame,eye_class,(int(left_right_eye[i][0])-eye_bias, int(left_right_eye[i][1])-bias),cv2.FONT_HERSHEY_COMPLEX,0.6,(0,255,0) \ ,thickness=2) else: eye_class = "other:{:.2f}".format( result_buff[i][1].cpu().item()) cv2.putText(frame,eye_class,(int(left_right_eye[i][0])-eye_bias, int(left_right_eye[i][1])-bias),cv2.FONT_HERSHEY_COMPLEX,0.6,(0,0,255) \ ,thickness=2) # bias += 30 eye_bias = 0 # left_eye left_eye_open, left_eye_close, left_eye_other = score_buff[0][ 0] cv2.putText(frame,"left_open:{:.2f}".format(left_eye_open) ,(10, 20),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_close:{:.2f}".format(left_eye_close) ,(10, 40),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_other:{:.2f}".format(left_eye_other) ,(10, 60),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) #right_eye right_eye_open, right_eye_close, right_eye_other = score_buff[ 1][0] cv2.putText(frame,"left_open:{:.2f}".format(right_eye_open) ,(200, 20),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_close:{:.2f}".format(right_eye_close) ,(200, 40),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_other:{:.2f}".format(right_eye_other) ,(200, 60),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) # eye_class = "open_eye" if 0 in t_result else "close_eye" img_name = os.path.split(img_p)[-1] cv2.imwrite(os.path.join(img_save, img_name), frame)
def dete_signal_video(): eye_class_dict = {0: "open_eye", 1: "close_eye", 2: "other"} point_nums = 24 threshold = [0.6, 0.7, 0.7] data_trans = Transforms.Compose([ Transforms.Resize((24, 24)), Transforms.ToTensor(), Transforms.Normalize((0.45, 0.448, 0.455), (0.082, 0.082, 0.082)), # Transforms.Normalize((0.407, 0.405, 0.412), (0.087, 0.087, 0.087)), ]) mixnet = MixNet(input_size=(24, 24), num_classes=3) # eye_class_dict = {0:"open_eye",1:"close_eye"} # weight_dict = torch.load("weight/signal_eye/Mixnet_epoch_29.pth") weight_dict = torch.load( "/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/weight/relabel_04_mix_SGD_mutillabel_24_24_20210302/Mixnet_epoch_49.pth" ) new_state_dict = OrderedDict() for k, v in weight_dict.items(): name = k[7:] new_state_dict[name] = v mixnet.load_state_dict(new_state_dict) # stat(net,(3,48,48)) mixnet.to('cuda:0') mixnet.eval() pnet, rnet, onet = create_mtcnn_net( p_model_path=r'model_store/final/pnet_epoch_19.pt', r_model_path=r'model_store/final/rnet_epoch_7.pt', o_model_path=r'model_store/final/onet_epoch_92.pt', use_cuda=True) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24, threshold=threshold) videos_root_path = 'test_video/hhh/02_65_6504_0_be4ba2aeac264ed992aae74c15b91b18.mp4' save_path_root = 'result_video/debug_test.avi' cap = cv2.VideoCapture(videos_root_path) fourcc = cv2.VideoWriter_fourcc(*'XVID') fps = cap.get(cv2.CAP_PROP_FPS) size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))) # tpa fname = os.path.splitext(os.path.split(videos_root_path)[1])[0] save_path = os.path.join( "/media/omnisky/D4T/JSH/faceFenlei/Projects/hul_eye_class/result_video/data(2)", fname + ".avi") out = cv2.VideoWriter(save_path_root, fourcc, fps, size) while True: ret, frame = cap.read() if ret: copy_frame = frame.copy() left_right_eye = [] bboxs, landmarks, wearmask = mtcnn_detector.detect_face(frame, rgb=True) temp_path, trmp_name = os.path.split(save_path) # trmp_name = os.path.splitext(trmp_name)[0] + "{:04d}.jpg".format(img_count) # tsave_path = os.path.join(temp_path, trmp_name) if landmarks is not None: eye_wild_buf = [] for i in range(landmarks.shape[0]): landmarks_one = landmarks[i, :] landmarks_one = landmarks_one.reshape((point_nums, 2)) left_eye = np.array(landmarks_one[[6, 8, 10, 11, 14], :]) xmin = np.min(left_eye[:, 0]) ymin = np.min(left_eye[:, 1]) xmax = np.max(left_eye[:, 0]) ymax = np.max(left_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) right_eye = np.array(landmarks_one[[7, 9, 12, 13, 15], :]) xmin = np.min(right_eye[:, 0]) ymin = np.min(right_eye[:, 1]) xmax = np.max(right_eye[:, 0]) ymax = np.max(right_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) #绘制眼睛点 # for j in [*left_eye,*right_eye]: # cv2.circle(frame, (int(j[0]), int(j[1])), 2, (255, 0, 0), -1) crop_img = [] for xmin, ymin, xmax, ymax in left_right_eye: w, h = xmax - xmin, ymax - ymin # 随机扩展大小0.05-0.15 k = 0.1 ratio = h / w if ratio > 1: ratio = ratio - 1 xmin -= (ratio / 2 * w + k * h) ymin -= (k * h) xmax += (ratio / 2 * w + k * h) ymax += (k * h) else: ratio = w / h - 1 xmin -= (k * w) ymin -= (ratio / 2 * h + k * w) xmax += (k * w) ymax += (ratio / 2 * h + k * w) eye_wild_buf.append(w) cv2.rectangle(frame, (int(xmin), int(ymin)), (int(xmax), int(ymax)), (0, 255, 255), 1) # 输出眼睛像素的长宽 temp_img = copy_frame[int(ymin):int(ymax), int(xmin):int(xmax)] # temp_img = cv2.resize(temp_img,(24,24)) crop_img.append(temp_img) if len(crop_img) < 2: cv2.imwrite(tsave_path, frame) # out.write(frame) continue # compose_img = np.hstack((crop_img[0],crop_img[1])) result_buff = [] score_buff = [] for i in crop_img: i = cv2.cvtColor(i, cv2.COLOR_BGR2RGB) t1 = time.time() compose_img = Image.fromarray(i) img = data_trans(compose_img) img = img.unsqueeze(0) with torch.no_grad(): outputs = mixnet(img.to('cuda:0')) spft_max = torch.nn.functional.softmax(outputs, dim=1) # 左眼右眼,分别三个类别的分数 score_buff.append(spft_max.cpu().numpy()) # 0,1->data,id score, result = torch.max(spft_max, 1) # result:最大值的id score:最大值的分数 result_buff.append([result.item(), score]) run_time = time.time() - t1 #0.005819 bias = 30 eye_bias = 100 for i in range(2): t_result = result_buff[i][0] #眼睛抠图的宽度 eye_w = eye_wild_buf[i] cv2.putText(frame,"w:{}".format(int(eye_w)),(int(left_right_eye[i][0])-eye_bias, int(left_right_eye[i][1])-50),cv2.FONT_HERSHEY_COMPLEX,0.6,(255,0,255) \ ,thickness=2) if 0 == t_result: # eye_class = "close_eye" # cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(0,255,0) \ # ,thickness=2) eye_class = "open_eye:{:.2f}".format( result_buff[i][1].cpu().item()) cv2.putText(frame,eye_class,(int(left_right_eye[i][0])-eye_bias, int(left_right_eye[i][1])-bias),cv2.FONT_HERSHEY_COMPLEX,0.6,(255,0,255) \ ,thickness=2) elif 1 == t_result: # eye_class = "open_eye" # cv2.putText(frame,eye_class,(int(xmax), int(ymax)-20),cv2.FONT_HERSHEY_COMPLEX,1.0,(255,0,255) \ # ,thickness=2) eye_class = "close_eye:{:.2f}".format( result_buff[i][1].cpu().item()) cv2.putText(frame,eye_class,(int(left_right_eye[i][0])-eye_bias, int(left_right_eye[i][1])-bias),cv2.FONT_HERSHEY_COMPLEX,0.6,(0,255,0) \ ,thickness=2) else: eye_class = "other:{:.2f}".format( result_buff[i][1].cpu().item()) cv2.putText(frame,eye_class,(int(left_right_eye[i][0])-eye_bias, int(left_right_eye[i][1])-bias),cv2.FONT_HERSHEY_COMPLEX,0.6,(0,0,255) \ ,thickness=2) # bias += 30 eye_bias = 0 # left_eye left_eye_open, left_eye_close, left_eye_other = score_buff[0][ 0] cv2.putText(frame,"left_open:{:.2f}".format(left_eye_open) ,(10, 20),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_close:{:.2f}".format(left_eye_close) ,(10, 40),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_other:{:.2f}".format(left_eye_other) ,(10, 60),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) #right_eye right_eye_open, right_eye_close, right_eye_other = score_buff[ 1][0] cv2.putText(frame,"left_open:{:.2f}".format(right_eye_open) ,(200, 20),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_close:{:.2f}".format(right_eye_close) ,(200, 40),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) cv2.putText(frame,"left_other:{:.2f}".format(right_eye_other) ,(200, 60),cv2.FONT_HERSHEY_COMPLEX,0.6,(20,150,0) \ ,thickness=2) # 计算最大概率的标号 max_id,max_score = (result_buff[0][0],result_buff[0][1].cpu().item()) if \ result_buff[0][1].cpu().item()>result_buff[1][1].cpu().item() else (result_buff[1][0],result_buff[1][1].cpu().item()) # 测试信息 eye_wild_buf_info = "w:[{:.2f},{:.2f}]".format( eye_wild_buf[0], eye_wild_buf[1]) # 测试时那个眼镜框最大 max_wilde_left_right = 0 if eye_wild_buf[0] > eye_wild_buf[1] else 1 # 获得最大宽度框的id和分数 # 宽度最大的 id 和分数 宽度第二大的 id和分数 max_wilde_id,max_wilde_score,max_wiled_second_id,max_wilde_second_score = (result_buff[0][0],result_buff[0][1].cpu().item(),result_buff[1][0],result_buff[1][1].cpu().item()) if \ max_wilde_left_right==0 else (result_buff[1][0],result_buff[1][1].cpu().item(),result_buff[0][0],result_buff[0][1].cpu().item()) score_buff_info = "score:[left: {:.2f}] [right: {:.2f}]".format( score_buff[0][0][2], score_buff[1][0][2]) cv2.putText(frame,eye_wild_buf_info,(400,80),cv2.FONT_HERSHEY_COMPLEX,0.6,(255,0,0) \ ,thickness=2) cv2.putText(frame,score_buff_info,(400,100),cv2.FONT_HERSHEY_COMPLEX,0.6,(255,0,0) \ ,thickness=2) # 如果 # if np.any(np.array(eye_wild_buf[:2])<19.0 )and max_score < 0.9 or np.any(np.array(eye_wild_buf[:2])<17.0 ) or np.any(np.array([score_buff[0][0][2],score_buff[1][0][2]])>= 0.5) and \ # max_score<0.9 or max_id==2: # 添加最大框 概率最大id=2 宽度最大的id=2 # if (eye_wild_buf[max_wilde_left_right]<17.0 ) or ((max_wilde_score>= 0.5) and \ # max_wilde_id==2 and max_wilde_second_score<0.85) or max_id==2 and (max_wilde_score < 0.8 and max_wilde_id != 2) or (max_id==2 and max_wilde_id == 2 and(max_wilde_second_score<0.8) ) or \ # (max_wilde_id == 2 and max_wiled_second_id==2 and (max_wilde_second_score>0.5 or max_wilde_score>0.5)) or ( eye_wild_buf[ 0 if max_wilde_left_right else 1]<17.0 ) or \ # ((eye_wild_buf[ 0 if max_wilde_left_right else 1]>23 and max_wilde_second_score>0.8 and max_wilde_id==2) or \ # (eye_wild_buf[max_wilde_left_right]>23 and max_wilde_score >0.8 and max_wiled_second_id==2)): # 左眼右眼宽度大于23 且概率大于0.8 且id=2 # 存在小于17像素的框且最大宽度的分数小于0.8 # 存在other概率大于0.5 # 存在小于10像素直接判断为other if ((eye_wild_buf[ 0 if max_wilde_left_right else 1]>23 and max_wilde_second_score>0.8 and max_wiled_second_id==2) or \ (eye_wild_buf[ max_wilde_left_right]>23 and max_wilde_score >0.8 and max_wilde_score==2) or \ (np.any(np.array(eye_wild_buf[:2])<17.0) and (max_wilde_score<0.8)) or ((max_wilde_id==2 and max_wilde_score>0.5 and max_wilde_second_score<0.9) or (max_wiled_second_id==2 and max_wilde_second_score>0.5 and max_wilde_score<0.9)) or\ (np.any(np.array(eye_wild_buf[:2])<10.0)) ): # 如果像素小于19且最大概率的眼睛小于0.9 或 任何一个像素小于12 且 max分数小于0.9 或 other # 2.任意一个other>=50 cv2.putText(frame,"other",(400,60),cv2.FONT_HERSHEY_COMPLEX,0.6,(0,0,255) \ ,thickness=2) # elif np.any(np.array([score_buff[0][0][1],score_buff[1][0][1]])>= 0.85) \ # or (max_id==1 and max_score>0.750): elif (max_wilde_id==1 and max_wilde_score>=0.80) \ or (max_id==1 and max_score>0.750): # elif (max_wilde_score >= 0.85) and max_wilde_id==1 \ # or (max_wilde_id==1 and max_wilde_score>0.750): # 任意一个闭眼概率大于0.9 # 最大值是闭眼且概率大于0.75 cv2.putText(frame,"close",(400,60),cv2.FONT_HERSHEY_COMPLEX,0.6,(0,255,0) \ ,thickness=2) else: cv2.putText(frame,"open",(400,60),cv2.FONT_HERSHEY_COMPLEX,0.6,(255,0,0) \ ,thickness=2) # cv2.imshow("frame",frame) out.write(frame) else: print("finish") break
cap = cv2.VideoCapture(0) #创建一个 VideoCapture 对象 pnet, rnet, onet = create_mtcnn_net( p_model_path="./original_model/pnet_epoch.pt", r_model_path="./original_model/rnet_epoch.pt", o_model_path="./original_model/onet_epoch.pt", use_cuda=True) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24) while (cap.isOpened()): #循环读取每一帧 bgr_image = cap.read()[1] bgr_image = cv2.flip(bgr_image, 1, dst=None) rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB) bboxs, landmarks = mtcnn_detector.detect_face(bgr_image) for cords in bboxs: x1, y1, x2, y2, s = cords bgr_image = cv2.rectangle(bgr_image, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0)) cv2.imshow('detection result', bgr_image) if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() #释放摄像头 cv2.destroyAllWindows() #删除建立的全部窗口
def test_face_alignment(self, test_moudel, savePic): pnet, rnet, onet_jiang = create_mtcnn_net( p_model_path="./original_model/pnet_epoch.pt", r_model_path="./original_model/rnet_epoch.pt", o_model_path="./original_model/" + test_moudel + ".pt", use_cuda=False) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet_jiang, min_face_size=24) train_data = TrainImageReader(self.gt_imdb, 48, batch_size=100, shuffle=False) # 读入1个batch的数据 # train_data.reset() total_errors = 0 cnt = 0 for i, (images, (gt_labels, gt_bboxes, gt_landmarks)) in enumerate(train_data): #取1个batch # if i == 12: # print(i) list_imgs = [images[i, :, :, :] for i in range(images.shape[0])] # 100张图片 list_bboxes = [gt_bboxes[i, :] for i in range(gt_bboxes.shape[0])] list_gt_landmarks = [ gt_landmarks[i, :] for i in range(gt_landmarks.shape[0]) ] mix = list(zip(list_imgs, list_bboxes, list_gt_landmarks)) batch_errors = [] for img, gt_bbox, gt_landmark in mix: # 取1个图片 bboxs, landmarks = mtcnn_detector.detect_face( img) # 原始的图片用原始网络检测 if landmarks.size: cnt += 1 bboxs = bboxs[:1] # 多个检测框保留第一个 landmarks = landmarks[:1] if savePic: vis_face(img, bboxs, landmarks, self.output_dir + str(cnt) + ".jpg") # 保存图片 gt_landmark = np.array(gt_landmark).reshape(5, 2) landmarks = np.array(landmarks).reshape(5, 2) normDist = np.linalg.norm(gt_landmark[1] - gt_landmark[0]) # 左右眼距离 error = np.mean( np.sqrt(np.sum( (landmarks - gt_landmark)**2, axis=1))) / normDist # print("the %sth pic error is : %s"%(cnt, error)) batch_errors.append(error) batch_errors = np.array(batch_errors).sum() total_errors += batch_errors print("%s: %s pics mean error is %s" % (datetime.datetime.now(), cnt, total_errors / cnt)) f = open("landmark_test.txt", "a+") f.write("%s, moudel_name:%s.pt, %s pics mean error is %s\n" % (datetime.datetime.now(), test_moudel, cnt, np.array(total_errors).reshape(1, -1).sum() / cnt)) f.close() print("%s:%s pics mean error is %s" % (datetime.datetime.now(), cnt, total_errors / cnt))
class VisSingleCase(object): def __init__(self, args): self.model = MtcnnDetector(args) def _fliter_doc_bbox(self, bboxes, landmarks): ''' Filter the face_box on card ''' area = (bboxes[:, 2] - bboxes[:, 0] + 1) * (bboxes[:, 3] - bboxes[:, 1] + 1) * -1 area_index = area.argsort() bbox = bboxes[area_index[0]] landmark = landmarks[area_index[0]] #prob_index = (bboxes[-1]*-1).argsort() # assist # if bboxes.shape[0] == 2 or area_index[0] == prob_index[0]: # bbox = bboxes[area_index[0]] # landmark = landmark[area_index[0]] return bbox, landmark def _fetch_block(self, bbox, landmark, lmk_flag=True): ''' Crop the chin_block of the detected face ''' if lmk_flag: landmark = landmark.reshape(-1, 2) left_down = landmark[6] # default : 6 nose_point = landmark[33] # point-34 | nose right_down = landmark[10] # default : 10 chin_point = landmark[8] # point-9 | chin x1, y1 = int(left_down[0]), int(nose_point[1]) x2, y2 = int(right_down[0]), int(chin_point[1]) else: x1, y1 = int(bbox[0]), int(bbox[1]) x2, y2 = int(bbox[2]), int(bbox[3]) return (x1, y1, x2, y2) def _vis_result(self, img, bbox, landmark): ''' Visual the detect-result and block ''' pil_img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) draw = ImageDraw.Draw(pil_img) block = self._fetch_block(bbox, landmark) draw.rectangle([(bbox[0], bbox[1]), (bbox[2], bbox[3])], outline='yellow') draw.rectangle([(block[0], block[1]), (block[2], block[3])], outline='red') pts = landmark.reshape((68, 2)) for idx in range(68): point = (pts[idx, 0] - 1, pts[idx, 1] - 1, pts[idx, 0] + 1, pts[idx, 1] + 1) draw.ellipse(point, fill=None, outline='green') pil_img.show() def check_case(self, img_path): ''' Check the single case ''' try: img = cv2.imread(img_path) bboxes, landmarks = self.model.detect_face(img, verbose=False) print((img_path, bboxes.shape)) except Exception as e: print(e) else: if bboxes.shape[0] == 0: print('No face detected in %s' % img_path) else: bbox, landmark = self._fliter_doc_bbox(bboxes, landmarks) self._vis_result(img, bbox, landmark)
def main(): eye_class_dict = {0: "open_eye", 1: "close_eye", 2: "other"} point_nums = 24 threshold = [0.6, 0.7, 0.7] data_trans = Transforms.Compose([ # Transforms.Resize((24, 48)), Transforms.ToTensor(), Transforms.Normalize((0.407, 0.405, 0.412), (0.087, 0.087, 0.087)), ]) mixnet = MixNet(input_size=(24, 48), num_classes=3) weight_dict = torch.load("weight/change_mix_data_0202/Mixnet_epoch_59.pth") new_state_dict = OrderedDict() for k, v in weight_dict.items(): name = k[7:] new_state_dict[name] = v mixnet.load_state_dict(new_state_dict) # stat(net,(3,48,48)) mixnet.to('cuda:0') mixnet.eval() pnet, rnet, onet = create_mtcnn_net( p_model_path=r'model_store/final/pnet_epoch_19.pt', r_model_path=r'model_store/final/rnet_epoch_7.pt', o_model_path=r'model_store/final/onet_epoch_92.pt', use_cuda=True) mtcnn_detector = MtcnnDetector(pnet=pnet, rnet=rnet, onet=onet, min_face_size=24, threshold=threshold) videos_root_path = 'test_video/20200522164730261_0.avi' save_path_root = 'result_video/20200522164730261_0.avi' cap = cv2.VideoCapture(videos_root_path) fourcc = cv2.VideoWriter_fourcc(*'XVID') fps = cap.get(cv2.CAP_PROP_FPS) size = (int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))) out = cv2.VideoWriter(save_path_root, fourcc, fps, size) while True: ret, frame = cap.read() if ret: copy_frame = frame.copy() left_right_eye = [] bboxs, landmarks, wearmask = mtcnn_detector.detect_face(frame, rgb=True) if landmarks is not None: for i in range(landmarks.shape[0]): landmarks_one = landmarks[i, :] landmarks_one = landmarks_one.reshape((point_nums, 2)) left_eye = np.array(landmarks_one[[6, 8, 10, 11, 14], :]) xmin = np.min(left_eye[:, 0]) ymin = np.min(left_eye[:, 1]) xmax = np.max(left_eye[:, 0]) ymax = np.max(left_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) right_eye = np.array(landmarks_one[[7, 9, 12, 13, 15], :]) xmin = np.min(right_eye[:, 0]) ymin = np.min(right_eye[:, 1]) xmax = np.max(right_eye[:, 0]) ymax = np.max(right_eye[:, 1]) left_right_eye.append([xmin, ymin, xmax, ymax]) # cv2.rectangle(frame,(int(xmin),int(ymin)),(int(xmax),int(ymax)),(0,255,0),2) for j in [*left_eye, *right_eye]: cv2.circle(frame, (int(j[0]), int(j[1])), 2, (255, 0, 0), -1) crop_img = [] for xmin, ymin, xmax, ymax in left_right_eye: w, h = xmax - xmin, ymax - ymin # 随机扩展大小0.05-0.15 k = 0.1 ratio = h / w if ratio > 1: ratio = ratio - 1 xmin -= (ratio / 2 * w + k * h) ymin -= (k * h) xmax += (ratio / 2 * w + k * h) ymax += (k * h) else: ratio = w / h - 1 xmin -= (k * w) ymin -= (ratio / 2 * h + k * w) xmax += (k * w) ymax += (ratio / 2 * h + k * w) cv2.rectangle(frame, (int(xmin), int(ymin)), (int(xmax), int(ymax)), (0, 255, 255), 2) temp_img = copy_frame[int(ymin):int(ymax), int(xmin):int(xmax)] temp_img = cv2.resize(temp_img, (24, 24)) crop_img.append(temp_img) if len(crop_img) < 2: out.write(frame) continue compose_img = np.hstack((crop_img[0], crop_img[1])) compose_img = cv2.cvtColor(compose_img, cv2.COLOR_BGR2RGB) compose_img = Image.fromarray(compose_img) img = data_trans(compose_img) img = img.unsqueeze(0) with torch.no_grad(): outputs = mixnet(img.to('cuda:0')) result = torch.max(outputs, 1)[1] eye_class = eye_class_dict[result.item()] cv2.putText(frame,eye_class,(0,20),cv2.FONT_HERSHEY_COMPLEX,1.3,(255,0,255) \ if result.item() == 0 else (255,255,0),thickness=2) out.write(frame) else: print("finish") break