def Rnet_Process(self, total_boxes, img):
'''
total_boxes: [[x1,y1,x2,y2,score]]
return: rectangles [[x1,y1,x2,y2,score]]
'''
height, width, _ = img.shape
num_box = total_boxes.shape[0]
# pad the bbox
[dy, edy, dx, edx, y, ey, x, ex, tmpw,
tmph] = self.pad(total_boxes, width, height)
# (3, 24, 24) is the input shape for RNet
input_buf = np.zeros((num_box, 3, 24, 24), dtype=np.float32)
for i in range(num_box):
tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.uint8)
tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1,
x[i]:ex[i] + 1, :]
input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (24, 24)))
output = self.RNet.predict(input_buf)
# filter the total_boxes with threshold
passed = np.where(output[1][:, 1] > self.threshold[1])
total_boxes = total_boxes[passed]
if total_boxes.size == 0:
return []
total_boxes[:, 4] = output[1][passed, 1].reshape((-1, ))
reg = output[0][passed]
# nms
pick = nms(total_boxes, 0.7, 'Union')
total_boxes = total_boxes[pick]
total_boxes = self.calibrate_box(total_boxes, reg[pick])
total_boxes = self.convert_to_square(total_boxes)
total_boxes[:, 0:4] = np.round(total_boxes[:, 0:4])
return total_boxes
def Lnet_Process(self, rectangles, img):
'''
rectangles: [boxes,score,points]
'''
height, width, _ = img.shape
total_boxes = rectangles[:, :5]
points = rectangles[:, 5:]
num_box = total_boxes.shape[0]
patchw = np.maximum(total_boxes[:, 2] - total_boxes[:, 0] + 1,
total_boxes[:, 3] - total_boxes[:, 1] + 1)
patchw = np.round(patchw * 0.25)
# make it even
patchw[np.where(np.mod(patchw, 2) == 1)] += 1
input_buf = np.zeros((num_box, 15, 24, 24), dtype=np.float32)
for i in range(5):
x, y = points[:, i], points[:, i + 5]
x, y = np.round(x - 0.5 * patchw), np.round(y - 0.5 * patchw)
[dy, edy, dx, edx, y, ey, x, ex, tmpw, tmph] = self.pad(
np.vstack([x, y, x + patchw - 1, y + patchw - 1]).T, width,
height)
for j in range(num_box):
tmpim = np.zeros((tmpw[j], tmpw[j], 3), dtype=np.float32)
tmpim[dy[j]:edy[j] + 1,
dx[j]:edx[j] + 1, :] = img[y[j]:ey[j] + 1,
x[j]:ex[j] + 1, :]
input_buf[j, i * 3:i * 3 + 3, :, :] = adjust_input(
cv2.resize(tmpim, (24, 24)))
output = self.LNet.predict(input_buf)
pointx = np.zeros((num_box, 5))
pointy = np.zeros((num_box, 5))
for k in range(5):
# do not make a large movement
tmp_index = np.where(np.abs(output[k] - 0.5) > 0.35)
output[k][tmp_index[0]] = 0.5
pointx[:, k] = np.round(points[:, k] -
0.5 * patchw) + output[k][:, 0] * patchw
pointy[:, k] = np.round(points[:, k + 5] -
0.5 * patchw) + output[k][:, 1] * patchw
points = np.hstack([pointx, pointy])
points = points.astype(np.int32)
points_xy = []
for j in range(5):
points_xy.append(points[:, j])
points_xy.append(points[:, j + 5])
points_xy = np.array(points_xy)
if config.x_y:
rectangles = np.concatenate((total_boxes, points_xy.T), axis=1)
else:
rectangles = np.concatenate((total_boxes, points), axis=1)
return rectangles
def Onet_Process(self, total_boxes, img):
'''
total_boxes: [[x1,y1,x2,y2,score]]
return: rectangles [[x1,y1,x2,y2,score]]
'''
num_box = total_boxes.shape[0]
height, width, _ = img.shape
# pad the bbox
[dy, edy, dx, edx, y, ey, x, ex, tmpw,
tmph] = self.pad(total_boxes, width, height)
# (3, 48, 48) is the input shape for ONet
input_buf = np.zeros((num_box, 3, 48, 48), dtype=np.float32)
for i in range(num_box):
tmp = np.zeros((tmph[i], tmpw[i], 3), dtype=np.float32)
tmp[dy[i]:edy[i] + 1, dx[i]:edx[i] + 1, :] = img[y[i]:ey[i] + 1,
x[i]:ex[i] + 1, :]
input_buf[i, :, :, :] = adjust_input(cv2.resize(tmp, (48, 48)))
output = self.ONet.predict(input_buf)
# filter the total_boxes with threshold
passed = np.where(output[2][:, 1] > self.threshold[2])
total_boxes = total_boxes[passed]
if total_boxes.size == 0:
return []
total_boxes[:, 4] = output[2][passed, 1].reshape((-1, ))
reg = output[1][passed]
points = output[0][passed]
# compute landmark points
bbw = total_boxes[:, 2] - total_boxes[:, 0] + 1
bbh = total_boxes[:, 3] - total_boxes[:, 1] + 1
points[:, 0:5] = np.expand_dims(
total_boxes[:, 0], 1) + np.expand_dims(bbw, 1) * points[:, 0:5]
points[:, 5:10] = np.expand_dims(
total_boxes[:, 1], 1) + np.expand_dims(bbh, 1) * points[:, 5:10]
# nms
total_boxes = self.calibrate_box(total_boxes, reg)
pick = nms(total_boxes, 0.3, 'Min')
total_boxes = total_boxes[pick]
points = points[pick]
points_xy = []
for j in range(5):
points_xy.append(points[:, j])
points_xy.append(points[:, j + 5])
points_xy = np.array(points_xy)
if cfg.x_y:
rectangles = np.concatenate((total_boxes, points_xy.T), axis=1)
else:
rectangles = np.concatenate((total_boxes, points), axis=1)
return rectangles