def detect_onet(self, im, dets, no_thresh=False):
"""Get face candidates using onet
Parameters:
----------
im: numpy array
input image array
dets: numpy array
detection results of rnet
Returns:
-------
boxes: numpy array
detected boxes before calibration
boxes_c: numpy array
boxes after calibration
"""
h, w, c = im.shape
dets = self.convert_to_square(dets)
dets[:, 0:4] = np.round(dets[:, 0:4])
[dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(dets, w, h)
num_boxes = dets.shape[0]
cropped_ims = np.zeros((num_boxes, 48, 48, 3), dtype=np.float32)
for i in range(num_boxes):
tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8)
tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = im[y[i]:ey[i] + 1,
x[i]:ex[i] + 1, :]
cropped_ims[i, :, :, :] = (cv2.resize(tmp, (48, 48)) - 127.5) / 128
cls_scores, reg, landmark = self.onet_detector.predict(cropped_ims)
# prob belongs to face
cls_scores = cls_scores[:, 1]
if not no_thresh:
keep_inds = np.where(cls_scores > self.thresh[2])[0]
else:
keep_inds = np.where(cls_scores > 0)[0]
if len(keep_inds) > 0:
# pickout filtered box
boxes = dets[keep_inds]
boxes[:, 4] = cls_scores[keep_inds]
reg = reg[keep_inds]
landmark = landmark[keep_inds]
else:
return None, None, None
# width
w = boxes[:, 2] - boxes[:, 0] + 1
# height
h = boxes[:, 3] - boxes[:, 1] + 1
landmark[:, 0::2] = (np.tile(w, (5, 1)) * landmark[:, 0::2].T +
np.tile(boxes[:, 0], (5, 1)) - 1).T
landmark[:, 1::2] = (np.tile(h, (5, 1)) * landmark[:, 1::2].T +
np.tile(boxes[:, 1], (5, 1)) - 1).T
boxes_c = self.calibrate_box(boxes, reg)
boxes = boxes[py_nms(boxes, 0.6, "Minimum")]
keep = py_nms(boxes_c, 0.6, "Minimum")
boxes_c = boxes_c[keep]
landmark = landmark[keep]
return boxes, boxes_c, landmark
def detect_rnet(self, im, dets):
"""Get face candidates using rnet
Parameters:
----------
im: numpy array
input image array
dets: numpy array
detection results of pnet
Returns:
-------
boxes: numpy array
detected boxes before calibration
boxes_c: numpy array
boxes after calibration
"""
h, w, c = im.shape
dets = self.convert_to_square(dets)
dets[:, 0:4] = np.round(dets[:, 0:4])
[dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(dets, w, h)
num_boxes = dets.shape[0]
cropped_ims = np.zeros((num_boxes, 24, 24, 3), dtype=np.float32)
for i in range(num_boxes):
tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8)
tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = im[y[i]:ey[i] + 1,
x[i]:ex[i] + 1, :]
cropped_ims[i, :, :, :] = (cv2.resize(tmp, (24, 24)) - 127.5) / 128
# cls_scores : num_data*2
# reg: num_data*4
# landmark: num_data*10
cls_scores, reg, _ = self.rnet_detector.predict(cropped_ims)
cls_scores = cls_scores[:, 1]
keep_inds = np.where(cls_scores > self.thresh[1])[0]
if len(keep_inds) > 0:
boxes = dets[keep_inds]
boxes[:, 4] = cls_scores[keep_inds]
reg = reg[keep_inds]
# landmark = landmark[keep_inds]
else:
return None, None, None
keep = nms.py_nms(boxes, 0.6)
boxes = boxes[keep]
boxes_c = self.calibrate_box(boxes, reg[keep])
return boxes, boxes_c, None
def detect_rnet(self, im, dets):
""" 获取PNet候选框区域图像进行RNet检测
:param im: 原始图像
:param dets: PNet检测的候选框
:return boxes: 通过长宽归一化后的候选框
:return boxes_c: 根据长宽恢复到原始图像尺度的候选框
"""
h, w, c = im.shape
dets = self.convert_to_square(dets)
dets[:, 0:4] = np.round(dets[:, 0:4])
[dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(dets, w, h)
num_boxes = dets.shape[0]
cropped_ims = np.zeros((num_boxes, 24, 24, 3), dtype=np.float32)
for i in range(num_boxes):
tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8)
tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = im[y[i]:ey[i] + 1,
x[i]:ex[i] + 1, :]
cropped_ims[i, :, :, :] = (cv2.resize(tmp, (24, 24)) - 127.5) / 128
# cls_scores : num_data*2
# reg: num_data*4
# landmark: num_data*10
cls_scores, reg, _ = self.rnet_detector.predict(cropped_ims)
cls_scores = cls_scores[:, 1]
keep_inds = np.where(cls_scores > self.thresh[1])[0]
if len(keep_inds) > 0:
boxes = dets[keep_inds]
boxes[:, 4] = cls_scores[keep_inds]
reg = reg[keep_inds]
# landmark = landmark[keep_inds]
else:
return None, None, None
# 先nms，再做offset矫正回归框
keep = py_nms(boxes, 0.6)
boxes = boxes[keep]
boxes_c = self.calibrate_box(boxes, reg[keep])
'''
# 先做offset矫正回归框，再做nms（RNet这个召回率高点）
boxes_c = self.calibrate_box(boxes, reg)
keep = py_nms(boxes_c, 0.6)
boxes = boxes[keep]
boxes_c = boxes_c[keep]
'''
return boxes, boxes_c, None
def detect_pnet(self, im):
"""Get face candidates through pnet
Parameters:
----------
im: numpy array
input image array
Returns:
-------
boxes: numpy array
detected boxes before calibration
boxes_c: numpy array
boxes after calibration
"""
# h, w, c = im.shape
net_size = 12
current_scale = float(
net_size) / self.min_face_size # find initial scale
# print("current_scale", net_size, self.min_face_size, current_scale)
im_resized = self.processed_image(im, current_scale)
current_height, current_width, _ = im_resized.shape
# fcn
all_boxes = list()
while min(current_height, current_width) > net_size:
# return the result predicted by pnet
# cls_cls_map : H*w*2
# reg: H*w*4
cls_cls_map, reg = self.pnet_detector.predict(im_resized)
# boxes: num*9(x1,y1,x2,y2,score,x1_offset,y1_offset,x2_offset,y2_offset)
boxes = self.generate_bbox(cls_cls_map[:, :, 1], reg,
current_scale, self.thresh[0])
current_scale *= self.scale_factor
im_resized = self.processed_image(im, current_scale)
current_height, current_width, _ = im_resized.shape
if boxes.size == 0:
continue
keep = nms.py_nms(boxes[:, :5], 0.5, 'Union')
boxes = boxes[keep]
all_boxes.append(boxes)
if len(all_boxes) == 0:
return None, None, None
all_boxes = np.vstack(all_boxes)
# merge the detection from first stage
keep = nms.py_nms(all_boxes[:, 0:5], 0.7, 'Union')
all_boxes = all_boxes[keep]
boxes = all_boxes[:, :5]
bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1
bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1
# refine the boxes
boxes_c = np.vstack([
all_boxes[:, 0] + all_boxes[:, 5] * bbw,
all_boxes[:, 1] + all_boxes[:, 6] * bbh,
all_boxes[:, 2] + all_boxes[:, 7] * bbw,
all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4]
])
boxes_c = boxes_c.T
return boxes, boxes_c, None
def detect_pnet(self, im):
""" 通过PNet获取单张图片的目标检测候选框
:param im: 图像数据
:return boxes: 通过长宽归一化后的候选框
:return boxes_c: 根据长宽恢复到原始图像尺度的候选框
"""
net_size = 12
# 初始化scale：将输入图像做尺度变换，以输入PNet进行检测
# 初始scale能检测最小尺度目标，后续scale逐渐变小，图片resize为更小尺度，PNet相对而言感受域变大，检测更大目标
current_scale = float(net_size) / self.min_face_size
# 根据初始尺度做图像缩小，以放入PNet做检测
im_resized = self.processed_image(im, current_scale)
current_height, current_width, _ = im_resized.shape
# 全卷积网络
all_boxes = list()
while min(current_height, current_width) > net_size:
# PNet预测：类别(H * W * 2)，bounding boxes(H * W * 4)
cls_map, detected_boxes = self.pnet_detector.predict(im_resized)
# bounding boxes恢复到原图像尺寸的坐标及off_set
# 预测的bounding boxes数量 * 9：x1, y1, x2, y2, score, x1_offset, y1_offset, x2_offset, y2_offset
boxes = self.generate_bbox(cls_map[:, :, 1], detected_boxes,
current_scale, self.thresh[0])
# 根据预设的尺度做图像缩小，以放入PNet做检测
current_scale *= self.scale_factor
im_resized = self.processed_image(im, current_scale)
current_height, current_width, _ = im_resized.shape
if boxes.size == 0:
continue
# 做非极大值抑制，筛选候选框
keep = py_nms(boxes[:, :5], 0.5, 'Union')
boxes = boxes[keep]
all_boxes.append(boxes)
if len(all_boxes) == 0:
return None, None, None
# 先nms，再做offset矫正回归框
# 对PNet所有尺度检测得到的候选框做非极大值抑制，筛选候选框
all_boxes = np.vstack(all_boxes)
keep = py_nms(all_boxes[:, 0:5], 0.7, 'Union')
all_boxes = all_boxes[keep]
boxes = all_boxes[:, :5]
# 根据offset计算实际预测的候选框坐标
bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1
bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1
boxes_c = np.vstack([
all_boxes[:, 0] + all_boxes[:, 5] * bbw,
all_boxes[:, 1] + all_boxes[:, 6] * bbh,
all_boxes[:, 2] + all_boxes[:, 7] * bbw,
all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4]
])
boxes_c = boxes_c.T
'''
# 先做offset矫正回归框，再做nms
all_boxes = np.vstack(all_boxes)
# 根据offset计算实际预测的候选框坐标
bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1
bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1
boxes_c = np.vstack([all_boxes[:, 0] + all_boxes[:, 5] * bbw,
all_boxes[:, 1] + all_boxes[:, 6] * bbh,
all_boxes[:, 2] + all_boxes[:, 7] * bbw,
all_boxes[:, 3] + all_boxes[:, 8] * bbh,
all_boxes[:, 4]])
boxes_c = boxes_c.T
# 对PNet所有尺度检测得到的候选框做非极大值抑制，筛选候选框
keep = py_nms(boxes_c[:, 0:5], 0.7, 'Union')
all_boxes = all_boxes[keep]
boxes = all_boxes[:, :5]
boxes_c = boxes_c[keep]
'''
return boxes, boxes_c, None
def detect_pnet(self, im):
"""Get face candidates through pnet
Parameters:
----------
im: numpy array
input image array
Returns:
-------
boxes: numpy array
detected boxes before calibration
boxes_c: numpy array
boxes after calibration
"""
h, w, c = im.shape
net_size = 12
current_scale = float(
net_size) / self.min_face_size # find initial scale
# print("current_scale", net_size, self.min_face_size, current_scale)
# risize image using current_scale
im_resized = self.processed_image(
im, current_scale,
round if self.pnet_detector.is_quantized else int)
current_height, current_width, _ = im_resized.shape
#print('current height and width:',current_height,current_width)
# fcn
# print('init img size = ', im.shape, ' current_scale = ', current_scale)
all_boxes = list()
while min(current_height, current_width) > net_size:
# return the result predicted by pnet
# cls_cls_map : H*w*2
# reg: H*w*4
# class_prob andd bbox_pred
# print ('pnet: detect resized = ', im_resized.shape)
cls_cls_map, reg = self.pnet_detector.predict(im_resized)
# print('cls_cls_map = ', cls_cls_map.shape)
# boxes: num*9(x1,y1,x2,y2,score,x1_offset,y1_offset,x2_offset,y2_offset)
boxes = self.generate_bbox(cls_cls_map[:, :, 1], reg,
current_scale, self.thresh[0])
# scale_factor is 0.79 in default
# print('gen boxes size = ', boxes.shape)
# if boxes.size > 0:
# print('map size ', cls_cls_map.shape, 'box size = ', boxes.shape , ' for size ', current_height)
# print('scores = ', boxes[:, 4])
current_scale *= self.scale_factor
im_resized = self.processed_image(
im, current_scale,
round if self.pnet_detector.is_quantized else int)
current_height, current_width, _ = im_resized.shape
if boxes.size == 0:
continue
# get the index from non-maximum s
keep = py_nms(boxes[:, :5], 0.5, 'Union')
boxes = boxes[keep]
# print ('append boxes = ', boxes.shape)
all_boxes.append(boxes)
if len(all_boxes) == 0:
return None, None, None
# print ('all_boxes len = ', len(all_boxes))
# for boxes in all_boxes:
# print('len box = ', boxes.shape)
all_boxes = np.vstack(all_boxes)
# merge the detection from first stage
# print ('input to nms shape= ', all_boxes.shape, 'shape 2 ', all_boxes[:, 0:5].shape)
keep = py_nms(all_boxes[:, 0:5], 0.7, 'Union')
# print ('box shape after nms = ', keep)
all_boxes = all_boxes[keep]
boxes = all_boxes[:, :5]
bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1
bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1
# refine the boxes
boxes_c = np.vstack([
all_boxes[:, 0] + all_boxes[:, 5] * bbw,
all_boxes[:, 1] + all_boxes[:, 6] * bbh,
all_boxes[:, 2] + all_boxes[:, 7] * bbw,
all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4]
])
boxes_c = boxes_c.T
# print('boxes shape', boxes.shape, ' box_c shape ', boxes_c.shape)
# print('boxes', boxes)
# print('boxes_c', boxes_c)
return boxes, boxes_c, None
