def Pnet_Process(self,img):
'''
input: mx_img, bgr order of shape (batch_size),h,w,3)
return: rectangles, [x1,y1,x2,y2,score]
'''
MIN_DET_SIZE = 12
# detected boxes
total_boxes = []
batch_size,height, width, _ = img.shape
minl = min(height, width)
# get all the valid scales
scales = []
m = MIN_DET_SIZE/self.minsize
minl *= m
factor_count = 0
while minl > MIN_DET_SIZE:
scales.append(m*self.factor**factor_count)
minl *= self.factor
factor_count += 1
# first stage
print("scal",len(scales))
t1 = time.time()
for s in scales:
local_boxes = detect_first_stage(img, self.PNet, s, self.threshold[0])
total_boxes.append(local_boxes)
if config.time:
print('Pnet inferince time: ',time.time()-t1)
# remove the empty
batch_boxes = []
empty_cnt = np.zeros(batch_size)
for i in range(batch_size):
tmp_box = []
for j in range(len(scales)):
if len(total_boxes[j][i])==0:
empty_cnt[i]+=1
else:
tmp_box.append(total_boxes[j][i])
if empty_cnt[i]==len(scales):
batch_boxes.append([])
else:
#print('tmp_box',np.shape(tmp_box),len(tmp_box),i,empty_cnt[i])
tmp_box = np.vstack(tmp_box)
pick = nms(tmp_box[:,:5], 0.7, 'Union')
tmp_box = tmp_box[pick]
#print(tmp_box.shape[0])
#tmp_box = self.get_topk(tmp_box)
bbw = tmp_box[:, 2] - tmp_box[:, 0] + 1
bbh = tmp_box[:, 3] - tmp_box[:, 1] + 1
# refine the bboxes
tmp_box = np.vstack([tmp_box[:, 0]+tmp_box[:, 5] * bbw,
tmp_box[:, 1]+tmp_box[:, 6] * bbh,
tmp_box[:, 2]+tmp_box[:, 7] * bbw,
tmp_box[:, 3]+tmp_box[:, 8] * bbh,
tmp_box[:, 4]
])
tmp_box = tmp_box.T
tmp_box = self.convert_to_square(tmp_box)
tmp_box[:, :4] = np.round(tmp_box[:, :4])
batch_boxes.append(tmp_box)
return batch_boxes
def Rnet_Process(self, total_boxes, img):
'''
total_boxes: [[x1,y1,x2,y2,score]]
return: rectangles [[x1,y1,x2,y2,score]]
'''
height, width, _ = img.shape
num_box = total_boxes.shape[0]
# pad the bbox
[dy, edy, dx, edx, y, ey, x, ex, tmpw,
tmph] = self.pad(total_boxes, width, height)
# (3, 24, 24) is the input shape for RNet
input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32)
for i in range(num_box):
tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8)
tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1,
x[i]:ex[i] + 1, :]
input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24)))
output = self.RNet.predict(input_buf)
# filter the total_boxes with threshold
passed = np.where(output[1][:, 1] > self.threshold[1])
total_boxes = total_boxes[passed]
if total_boxes.size == 0:
return []
total_boxes[:, 4] = output[1][passed, 1].reshape((-1, ))
reg = output[0][passed]
# nms
pick = nms(total_boxes, 0.7, 'Union')
total_boxes = total_boxes[pick]
total_boxes = self.calibrate_box(total_boxes, reg[pick])
total_boxes = self.convert_to_square(total_boxes)
total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4])
return total_boxes
def Pnet_Process(self, img):
'''
input: mx_img, bgr order of shape (1, 3, n, m)
return: rectangles, [x1,y1,x2,y2,score]
'''
MIN_DET_SIZE = 12
# detected boxes
total_boxes = []
height, width, _ = img.shape
minl = min(height, width)
# get all the valid scales
scales = []
m = MIN_DET_SIZE / self.minsize
minl *= m
factor_count = 0
while minl > MIN_DET_SIZE:
scales.append(m * self.factor**factor_count)
minl *= self.factor
factor_count += 1
# first stage
total_boxes = []
for s in scales:
local_boxes = detect_first_stage(img, self.PNet, s,
self.threshold[0])
if len(local_boxes) == 0:
continue
total_boxes.extend(local_boxes)
# remove the Nones
total_boxes = [i for i in total_boxes if i is not None]
if len(total_boxes) == 0:
return []
total_boxes = np.vstack(total_boxes)
if total_boxes.size == 0:
return []
# merge the detection from first stage
pick = nms(total_boxes[:, 0:5], 0.7, 'Union')
total_boxes = total_boxes[pick]
bbw = total_boxes[:, 2] - total_boxes[:, 0] + 1
bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1
# refine the bboxes
total_boxes = np.vstack([
total_boxes[:, 0] + total_boxes[:, 5] * bbw,
total_boxes[:, 1] + total_boxes[:, 6] * bbh,
total_boxes[:, 2] + total_boxes[:, 7] * bbw,
total_boxes[:, 3] + total_boxes[:, 8] * bbh, total_boxes[:, 4]
])
total_boxes = total_boxes.T
total_boxes = self.convert_to_square(total_boxes)
total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4])
return total_boxes
def Onet_Process(self, total_boxes, img):
'''
total_boxes: [[x1,y1,x2,y2,score]]
return: rectangles [[x1,y1,x2,y2,score]]
'''
num_box = total_boxes.shape[0]
height, width, _ = img.shape
# pad the bbox
[dy, edy, dx, edx, y, ey, x, ex, tmpw,
tmph] = self.pad(total_boxes, width, height)
# (3, 48, 48) is the input shape for ONet
input_buf = np.zeros((num_box, 3, 48, 48), dtype=np.float32)
for i in range(num_box):
tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.float32)
tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1,
x[i]:ex[i] + 1, :]
input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (48, 48)))
output = self.ONet.predict(input_buf)
# filter the total_boxes with threshold
passed = np.where(output[2][:, 1] > self.threshold[2])
total_boxes = total_boxes[passed]
if total_boxes.size == 0:
return []
total_boxes[:, 4] = output[2][passed, 1].reshape((-1, ))
reg = output[1][passed]
points = output[0][passed]
# compute landmark points
bbw = total_boxes[:, 2] - total_boxes[:, 0] + 1
bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1
points[:, 0:5] = np.expand_dims(
total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5]
points[:, 5:10] = np.expand_dims(
total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10]
# nms
total_boxes = self.calibrate_box(total_boxes, reg)
pick = nms(total_boxes, 0.3, 'Min')
total_boxes = total_boxes[pick]
points = points[pick]
points_xy = []
for j in range(5):
points_xy.append(points[:, j])
points_xy.append(points[:, j + 5])
points_xy = np.array(points_xy)
if cfg.x_y:
rectangles = np.concatenate((total_boxes, points_xy.T), axis=1)
else:
rectangles = np.concatenate((total_boxes, points), axis=1)
return rectangles