def detect_onet(self, im, dets, no_thresh=False): """Get face candidates using onet Parameters: ---------- im: numpy array input image array dets: numpy array detection results of rnet Returns: ------- boxes: numpy array detected boxes before calibration boxes_c: numpy array boxes after calibration """ h, w, c = im.shape dets = self.convert_to_square(dets) dets[:, 0:4] = np.round(dets[:, 0:4]) [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(dets, w, h) num_boxes = dets.shape[0] cropped_ims = np.zeros((num_boxes, 48, 48, 3), dtype=np.float32) for i in range(num_boxes): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = im[y[i]:ey[i] + 1, x[i]:ex[i] + 1, :] cropped_ims[i, :, :, :] = (cv2.resize(tmp, (48, 48)) - 127.5) / 128 cls_scores, reg, landmark = self.onet_detector.predict(cropped_ims) # prob belongs to face cls_scores = cls_scores[:, 1] if not no_thresh: keep_inds = np.where(cls_scores > self.thresh[2])[0] else: keep_inds = np.where(cls_scores > 0)[0] if len(keep_inds) > 0: # pickout filtered box boxes = dets[keep_inds] boxes[:, 4] = cls_scores[keep_inds] reg = reg[keep_inds] landmark = landmark[keep_inds] else: return None, None, None # width w = boxes[:, 2] - boxes[:, 0] + 1 # height h = boxes[:, 3] - boxes[:, 1] + 1 landmark[:, 0::2] = (np.tile(w, (5, 1)) * landmark[:, 0::2].T + np.tile(boxes[:, 0], (5, 1)) - 1).T landmark[:, 1::2] = (np.tile(h, (5, 1)) * landmark[:, 1::2].T + np.tile(boxes[:, 1], (5, 1)) - 1).T boxes_c = self.calibrate_box(boxes, reg) boxes = boxes[py_nms(boxes, 0.6, "Minimum")] keep = py_nms(boxes_c, 0.6, "Minimum") boxes_c = boxes_c[keep] landmark = landmark[keep] return boxes, boxes_c, landmark
def detect_rnet(self, im, dets): """Get face candidates using rnet Parameters: ---------- im: numpy array input image array dets: numpy array detection results of pnet Returns: ------- boxes: numpy array detected boxes before calibration boxes_c: numpy array boxes after calibration """ h, w, c = im.shape dets = self.convert_to_square(dets) dets[:, 0:4] = np.round(dets[:, 0:4]) [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(dets, w, h) num_boxes = dets.shape[0] cropped_ims = np.zeros((num_boxes, 24, 24, 3), dtype=np.float32) for i in range(num_boxes): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = im[y[i]:ey[i] + 1, x[i]:ex[i] + 1, :] cropped_ims[i, :, :, :] = (cv2.resize(tmp, (24, 24)) - 127.5) / 128 # cls_scores : num_data*2 # reg: num_data*4 # landmark: num_data*10 cls_scores, reg, _ = self.rnet_detector.predict(cropped_ims) cls_scores = cls_scores[:, 1] keep_inds = np.where(cls_scores > self.thresh[1])[0] if len(keep_inds) > 0: boxes = dets[keep_inds] boxes[:, 4] = cls_scores[keep_inds] reg = reg[keep_inds] # landmark = landmark[keep_inds] else: return None, None, None keep = nms.py_nms(boxes, 0.6) boxes = boxes[keep] boxes_c = self.calibrate_box(boxes, reg[keep]) return boxes, boxes_c, None
def detect_rnet(self, im, dets): """ 获取PNet候选框区域图像进行RNet检测 :param im: 原始图像 :param dets: PNet检测的候选框 :return boxes: 通过长宽归一化后的候选框 :return boxes_c: 根据长宽恢复到原始图像尺度的候选框 """ h, w, c = im.shape dets = self.convert_to_square(dets) dets[:, 0:4] = np.round(dets[:, 0:4]) [dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(dets, w, h) num_boxes = dets.shape[0] cropped_ims = np.zeros((num_boxes, 24, 24, 3), dtype=np.float32) for i in range(num_boxes): tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8) tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = im[y[i]:ey[i] + 1, x[i]:ex[i] + 1, :] cropped_ims[i, :, :, :] = (cv2.resize(tmp, (24, 24)) - 127.5) / 128 # cls_scores : num_data*2 # reg: num_data*4 # landmark: num_data*10 cls_scores, reg, _ = self.rnet_detector.predict(cropped_ims) cls_scores = cls_scores[:, 1] keep_inds = np.where(cls_scores > self.thresh[1])[0] if len(keep_inds) > 0: boxes = dets[keep_inds] boxes[:, 4] = cls_scores[keep_inds] reg = reg[keep_inds] # landmark = landmark[keep_inds] else: return None, None, None # 先nms,再做offset矫正回归框 keep = py_nms(boxes, 0.6) boxes = boxes[keep] boxes_c = self.calibrate_box(boxes, reg[keep]) ''' # 先做offset矫正回归框,再做nms(RNet这个召回率高点) boxes_c = self.calibrate_box(boxes, reg) keep = py_nms(boxes_c, 0.6) boxes = boxes[keep] boxes_c = boxes_c[keep] ''' return boxes, boxes_c, None
def detect_pnet(self, im): """Get face candidates through pnet Parameters: ---------- im: numpy array input image array Returns: ------- boxes: numpy array detected boxes before calibration boxes_c: numpy array boxes after calibration """ # h, w, c = im.shape net_size = 12 current_scale = float( net_size) / self.min_face_size # find initial scale # print("current_scale", net_size, self.min_face_size, current_scale) im_resized = self.processed_image(im, current_scale) current_height, current_width, _ = im_resized.shape # fcn all_boxes = list() while min(current_height, current_width) > net_size: # return the result predicted by pnet # cls_cls_map : H*w*2 # reg: H*w*4 cls_cls_map, reg = self.pnet_detector.predict(im_resized) # boxes: num*9(x1,y1,x2,y2,score,x1_offset,y1_offset,x2_offset,y2_offset) boxes = self.generate_bbox(cls_cls_map[:, :, 1], reg, current_scale, self.thresh[0]) current_scale *= self.scale_factor im_resized = self.processed_image(im, current_scale) current_height, current_width, _ = im_resized.shape if boxes.size == 0: continue keep = nms.py_nms(boxes[:, :5], 0.5, 'Union') boxes = boxes[keep] all_boxes.append(boxes) if len(all_boxes) == 0: return None, None, None all_boxes = np.vstack(all_boxes) # merge the detection from first stage keep = nms.py_nms(all_boxes[:, 0:5], 0.7, 'Union') all_boxes = all_boxes[keep] boxes = all_boxes[:, :5] bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1 bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1 # refine the boxes boxes_c = np.vstack([ all_boxes[:, 0] + all_boxes[:, 5] * bbw, all_boxes[:, 1] + all_boxes[:, 6] * bbh, all_boxes[:, 2] + all_boxes[:, 7] * bbw, all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4] ]) boxes_c = boxes_c.T return boxes, boxes_c, None
def detect_pnet(self, im): """ 通过PNet获取单张图片的目标检测候选框 :param im: 图像数据 :return boxes: 通过长宽归一化后的候选框 :return boxes_c: 根据长宽恢复到原始图像尺度的候选框 """ net_size = 12 # 初始化scale:将输入图像做尺度变换,以输入PNet进行检测 # 初始scale能检测最小尺度目标,后续scale逐渐变小,图片resize为更小尺度,PNet相对而言感受域变大,检测更大目标 current_scale = float(net_size) / self.min_face_size # 根据初始尺度做图像缩小,以放入PNet做检测 im_resized = self.processed_image(im, current_scale) current_height, current_width, _ = im_resized.shape # 全卷积网络 all_boxes = list() while min(current_height, current_width) > net_size: # PNet预测:类别(H * W * 2),bounding boxes(H * W * 4) cls_map, detected_boxes = self.pnet_detector.predict(im_resized) # bounding boxes恢复到原图像尺寸的坐标及off_set # 预测的bounding boxes数量 * 9:x1, y1, x2, y2, score, x1_offset, y1_offset, x2_offset, y2_offset boxes = self.generate_bbox(cls_map[:, :, 1], detected_boxes, current_scale, self.thresh[0]) # 根据预设的尺度做图像缩小,以放入PNet做检测 current_scale *= self.scale_factor im_resized = self.processed_image(im, current_scale) current_height, current_width, _ = im_resized.shape if boxes.size == 0: continue # 做非极大值抑制,筛选候选框 keep = py_nms(boxes[:, :5], 0.5, 'Union') boxes = boxes[keep] all_boxes.append(boxes) if len(all_boxes) == 0: return None, None, None # 先nms,再做offset矫正回归框 # 对PNet所有尺度检测得到的候选框做非极大值抑制,筛选候选框 all_boxes = np.vstack(all_boxes) keep = py_nms(all_boxes[:, 0:5], 0.7, 'Union') all_boxes = all_boxes[keep] boxes = all_boxes[:, :5] # 根据offset计算实际预测的候选框坐标 bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1 bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1 boxes_c = np.vstack([ all_boxes[:, 0] + all_boxes[:, 5] * bbw, all_boxes[:, 1] + all_boxes[:, 6] * bbh, all_boxes[:, 2] + all_boxes[:, 7] * bbw, all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4] ]) boxes_c = boxes_c.T ''' # 先做offset矫正回归框,再做nms all_boxes = np.vstack(all_boxes) # 根据offset计算实际预测的候选框坐标 bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1 bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1 boxes_c = np.vstack([all_boxes[:, 0] + all_boxes[:, 5] * bbw, all_boxes[:, 1] + all_boxes[:, 6] * bbh, all_boxes[:, 2] + all_boxes[:, 7] * bbw, all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4]]) boxes_c = boxes_c.T # 对PNet所有尺度检测得到的候选框做非极大值抑制,筛选候选框 keep = py_nms(boxes_c[:, 0:5], 0.7, 'Union') all_boxes = all_boxes[keep] boxes = all_boxes[:, :5] boxes_c = boxes_c[keep] ''' return boxes, boxes_c, None
def detect_pnet(self, im): """Get face candidates through pnet Parameters: ---------- im: numpy array input image array Returns: ------- boxes: numpy array detected boxes before calibration boxes_c: numpy array boxes after calibration """ h, w, c = im.shape net_size = 12 current_scale = float( net_size) / self.min_face_size # find initial scale # print("current_scale", net_size, self.min_face_size, current_scale) # risize image using current_scale im_resized = self.processed_image( im, current_scale, round if self.pnet_detector.is_quantized else int) current_height, current_width, _ = im_resized.shape #print('current height and width:',current_height,current_width) # fcn # print('init img size = ', im.shape, ' current_scale = ', current_scale) all_boxes = list() while min(current_height, current_width) > net_size: # return the result predicted by pnet # cls_cls_map : H*w*2 # reg: H*w*4 # class_prob andd bbox_pred # print ('pnet: detect resized = ', im_resized.shape) cls_cls_map, reg = self.pnet_detector.predict(im_resized) # print('cls_cls_map = ', cls_cls_map.shape) # boxes: num*9(x1,y1,x2,y2,score,x1_offset,y1_offset,x2_offset,y2_offset) boxes = self.generate_bbox(cls_cls_map[:, :, 1], reg, current_scale, self.thresh[0]) # scale_factor is 0.79 in default # print('gen boxes size = ', boxes.shape) # if boxes.size > 0: # print('map size ', cls_cls_map.shape, 'box size = ', boxes.shape , ' for size ', current_height) # print('scores = ', boxes[:, 4]) current_scale *= self.scale_factor im_resized = self.processed_image( im, current_scale, round if self.pnet_detector.is_quantized else int) current_height, current_width, _ = im_resized.shape if boxes.size == 0: continue # get the index from non-maximum s keep = py_nms(boxes[:, :5], 0.5, 'Union') boxes = boxes[keep] # print ('append boxes = ', boxes.shape) all_boxes.append(boxes) if len(all_boxes) == 0: return None, None, None # print ('all_boxes len = ', len(all_boxes)) # for boxes in all_boxes: # print('len box = ', boxes.shape) all_boxes = np.vstack(all_boxes) # merge the detection from first stage # print ('input to nms shape= ', all_boxes.shape, 'shape 2 ', all_boxes[:, 0:5].shape) keep = py_nms(all_boxes[:, 0:5], 0.7, 'Union') # print ('box shape after nms = ', keep) all_boxes = all_boxes[keep] boxes = all_boxes[:, :5] bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1 bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1 # refine the boxes boxes_c = np.vstack([ all_boxes[:, 0] + all_boxes[:, 5] * bbw, all_boxes[:, 1] + all_boxes[:, 6] * bbh, all_boxes[:, 2] + all_boxes[:, 7] * bbw, all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4] ]) boxes_c = boxes_c.T # print('boxes shape', boxes.shape, ' box_c shape ', boxes_c.shape) # print('boxes', boxes) # print('boxes_c', boxes_c) return boxes, boxes_c, None