def __detect_model(self,
model,
image,
cboxes,
thres_c,
thres_i,
size,
nms_mode='union'):
"""detection of Refinement Network and Output Network. """
bboxes = utils.convert_to_square(cboxes)
img_data = self.__crop_boxes(image, bboxes, size)
img_data = img_data.to(self.__device)
_cls, _offset, _landmark = model(img_data)
cls, offset, landmark = _cls[:, 0].cpu().data, _offset.cpu(
).data, _landmark.cpu().data
mask = torch.gt(cls, thres_c)
confidences, loffset, boffset = cls[mask], landmark[mask], offset[mask]
if not any(mask):
return confidences, loffset, boffset
boxes = utils.cord_regression(bboxes[mask], boffset)
landmarks = utils.cord_regression(bboxes[mask], loffset)
keep = utils.nms(boxes, confidences, threshold=thres_i, mode=nms_mode)
return confidences[keep], boxes[keep], landmarks[keep]
def gen_onet_sample_data(data_dir, anno_file, det_boxs_file, prefix):
neg_save_dir = os.path.join(data_dir, "48/negative")
pos_save_dir = os.path.join(data_dir, "48/positive")
part_save_dir = os.path.join(data_dir, "48/part")
for dir_path in [neg_save_dir, pos_save_dir, part_save_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# load ground truth from annotation file
# format of each line: image/path [x1,y1,x2,y2] for each gt_box in this image
with open(anno_file, 'r') as f:
annotations = f.readlines()
image_size = 48
net = "onet"
im_idx_list = list()
gt_boxes_list = list()
num_of_images = len(annotations)
print("processing %d images in total" % num_of_images)
for annotation in annotations:
annotation = annotation.strip().split(' ')
im_idx = os.path.join(prefix, annotation[0])
boxes = list(map(float, annotation[1:]))
boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4)
im_idx_list.append(im_idx)
gt_boxes_list.append(boxes)
save_path = './anno_store'
if not os.path.exists(save_path):
os.makedirs(save_path)
f1 = open(os.path.join(save_path, 'pos_%d.txt' % image_size), 'w')
f2 = open(os.path.join(save_path, 'neg_%d.txt' % image_size), 'w')
f3 = open(os.path.join(save_path, 'part_%d.txt' % image_size), 'w')
det_handle = open(det_boxs_file, 'rb')
det_boxes = cPickle.load(det_handle)
print(len(det_boxes), num_of_images)
# assert len(det_boxes) == num_of_images, "incorrect detections or ground truths"
# index of neg, pos and part face, used as their image names
n_idx = 0
p_idx = 0
d_idx = 0
image_done = 0
for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list):
if image_done % 100 == 0:
print("%d images done" % image_done)
image_done += 1
if dets.shape[0] == 0:
continue
img = cv2.imread(im_idx)
dets = convert_to_square(dets)
dets[:, 0:4] = np.round(dets[:, 0:4])
for box in dets:
x_left, y_top, x_right, y_bottom = box[0:4].astype(int)
width = x_right - x_left + 1
height = y_bottom - y_top + 1
# ignore box that is too small or beyond image border
if width < 20 or x_left < 0 or y_top < 0 or x_right > img.shape[
1] - 1 or y_bottom > img.shape[0] - 1:
continue
# compute intersection over union(IoU) between current box and all gt boxes
Iou = IoU(box, gts)
cropped_im = img[y_top:y_bottom + 1, x_left:x_right + 1, :]
resized_im = cv2.resize(cropped_im, (image_size, image_size),
interpolation=cv2.INTER_LINEAR)
# save negative images and write label
if np.max(Iou) < 0.3:
# Iou with all gts must below 0.3
save_file = os.path.join(neg_save_dir, "%s.jpg" % n_idx)
f2.write(save_file + ' 0\n')
cv2.imwrite(save_file, resized_im)
n_idx += 1
else:
# find gt_box with the highest iou
idx = np.argmax(Iou)
assigned_gt = gts[idx]
x1, y1, x2, y2 = assigned_gt
# compute bbox reg label
offset_x1 = (x1 - x_left) / float(width)
offset_y1 = (y1 - y_top) / float(height)
offset_x2 = (x2 - x_right) / float(width)
offset_y2 = (y2 - y_bottom) / float(height)
# save positive and part-face images and write labels
if np.max(Iou) >= 0.65:
save_file = os.path.join(pos_save_dir, "%s.jpg" % p_idx)
f1.write(save_file + ' 1 %.2f %.2f %.2f %.2f\n' %
(offset_x1, offset_y1, offset_x2, offset_y2))
cv2.imwrite(save_file, resized_im)
p_idx += 1
elif np.max(Iou) >= 0.4:
save_file = os.path.join(part_save_dir, "%s.jpg" % d_idx)
f3.write(save_file + ' -1 %.2f %.2f %.2f %.2f\n' %
(offset_x1, offset_y1, offset_x2, offset_y2))
cv2.imwrite(save_file, resized_im)
d_idx += 1
f1.close()
f2.close()
f3.close()
def gen_rnet_sample_data(data_dir, anno_dir, det_boxs_file):
''' Generate the train data for RNet '''
part_save_dir = os.path.join(data_dir, "5keypoints/rnet/part")
pos_save_dir = os.path.join(data_dir, "5keypoints/rnet/positive")
neg_save_dir = os.path.join(data_dir, "5keypoints/rnet/negative")
for dir_path in [neg_save_dir, pos_save_dir, part_save_dir]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
# load ground truth from annotation file
# format of each line: image/path [x1,y1,x2,y2] for each gt_box in this image
anno_file = os.path.join(anno_dir, 'anno_store/wide_anno_train.txt'
) # TODO :: [local_wide_anno, wide_anno_train]
with open(anno_file, 'r') as f:
annotations = f.readlines()
net, image_size = "rnet", 24
num_of_images = len(annotations)
im_idx_list, gt_boxes_list = [], []
print("processing %d images in total" % num_of_images)
for annotation in annotations:
annotation = annotation.strip().split(' ')
im_idx = os.path.join('', annotation[0])
# im_idx = annotation[0]
boxes = list(map(float, annotation[1:]))
boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4)
im_idx_list.append(im_idx)
gt_boxes_list.append(boxes)
save_path = os.path.join(anno_dir, 'anno_store/rnet')
if not os.path.exists(save_path):
os.makedirs(save_path)
f1 = open(os.path.join(save_path, 'pos_%d.txt' % image_size), 'w')
f2 = open(os.path.join(save_path, 'neg_%d.txt' % image_size), 'w')
f3 = open(os.path.join(save_path, 'part_%d.txt' % image_size), 'w')
# print(det_boxs_file)
det_handle = open(det_boxs_file, 'rb')
det_boxes = cPickle.load(det_handle)
print(len(det_boxes), num_of_images)
n_idx, p_idx, d_idx = 0, 0, 0
image_done = 0
for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list):
gts = np.array(gts, dtype=np.float32).reshape(-1, 4)
if image_done % 100 == 0:
print("%d images done" % image_done)
image_done += 1
if dets.shape[0] == 0:
continue
img = cv2.imread(im_idx)
# change to square
dets = convert_to_square(dets)
dets[:, 0:4] = np.round(dets[:, 0:4])
neg_num = 0
for box in dets:
x_left, y_top, x_right, y_bottom, _ = box.astype(int)
width = x_right - x_left + 1
height = y_bottom - y_top + 1
# ignore box that is too small or beyond image border
if width < 20 or x_left < 0 or y_top < 0 or \
(x_right > img.shape[1] - 1) or (y_bottom > img.shape[0] - 1):
continue
# compute intersection over union(IoU) between current box and all gt boxes
Iou = IoU(box, gts)
cropped_im = img[y_top:y_bottom + 1, x_left:x_right + 1, :]
resized_im = cv2.resize(cropped_im, (image_size, image_size),
interpolation=cv2.INTER_LINEAR)
# save negative images and write label
# Iou with all gts must below 0.3
if np.max(Iou) < 0.3 and neg_num < 60:
# save the examples
save_file = os.path.join(neg_save_dir, "%s.jpg" % n_idx)
# print(save_file)
f2.write(save_file + ' 0\n')
cv2.imwrite(save_file, resized_im)
n_idx += 1
neg_num += 1
else:
# find gt_box with the highest iou
idx = np.argmax(Iou)
assigned_gt = gts[idx]
x1, y1, x2, y2 = assigned_gt
# compute bbox reg label
offset_x1 = (x1 - x_left) / float(width)
offset_y1 = (y1 - y_top) / float(height)
offset_x2 = (x2 - x_right) / float(width)
offset_y2 = (y2 - y_bottom) / float(height)
# save positive and part-face images and write labels
if np.max(Iou) >= 0.65:
save_file = os.path.join(pos_save_dir, "%s.jpg" % p_idx)
f1.write(save_file + ' 1 %.2f %.2f %.2f %.2f\n' %
(offset_x1, offset_y1, offset_x2, offset_y2))
cv2.imwrite(save_file, resized_im)
p_idx += 1
elif np.max(Iou) >= 0.4:
save_file = os.path.join(part_save_dir, "%s.jpg" % d_idx)
f3.write(save_file + ' -1 %.2f %.2f %.2f %.2f\n' %
(offset_x1, offset_y1, offset_x2, offset_y2))
cv2.imwrite(save_file, resized_im)
d_idx += 1
f1.close()
f2.close()
f3.close()
def gen_sample_data(data_dir, anno_file, det_boxs_file, prefix_path=''):
landmark_save_dir = os.path.join(data_dir, "48/landmark")
if not os.path.exists(landmark_save_dir):
os.makedirs(landmark_save_dir)
# load ground truth from annotation file
# format of each line: image/path [x1,y1,x2,y2] for each gt_box in this image
with open(anno_file, 'r') as f:
annotations = f.readlines()
image_size = 48
net = "onet"
im_idx_list = list()
gt_boxes_list = list()
gt_landmark_list = list()
num_of_images = len(annotations)
print("processing %d images in total" % num_of_images)
for annotation in annotations:
annotation = annotation.strip().split(' ')
im_idx = annotation[0]
boxes = map(float, annotation[1:5])
boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4)
landmarks = map(float, annotation[5:])
landmarks = np.array(landmarks, dtype=np.float32).reshape(-1, 10)
im_idx_list.append(im_idx)
gt_boxes_list.append(boxes)
gt_landmark_list.append(landmarks)
save_path = config.ANNO_STORE_DIR
if not os.path.exists(save_path):
os.makedirs(save_path)
f = open(os.path.join(save_path, 'landmark_48.txt'), 'w')
det_handle = open(det_boxs_file, 'r')
det_boxes = cPickle.load(det_handle)
print(len(det_boxes), num_of_images)
assert len(
det_boxes) == num_of_images, "incorrect detections or ground truths"
# index of neg, pos and part face, used as their image names
p_idx = 0
image_done = 0
for im_idx, dets, gts, landmark in zip(im_idx_list, det_boxes,
gt_boxes_list, gt_landmark_list):
if image_done % 100 == 0:
print("%d images done" % image_done)
image_done += 1
if dets.shape[0] == 0:
continue
img = cv2.imread(os.path.join(prefix_path, im_idx))
dets = convert_to_square(dets)
dets[:, 0:4] = np.round(dets[:, 0:4])
for box in dets:
x_left, y_top, x_right, y_bottom = box[0:4].astype(int)
width = x_right - x_left + 1
height = y_bottom - y_top + 1
# ignore box that is too small or beyond image border
if width < 20 or x_left < 0 or y_top < 0 or x_right > img.shape[
1] - 1 or y_bottom > img.shape[0] - 1:
continue
# compute intersection over union(IoU) between current box and all gt boxes
Iou = IoU(box, gts)
cropped_im = img[y_top:y_bottom + 1, x_left:x_right + 1, :]
resized_im = cv2.resize(cropped_im, (image_size, image_size),
interpolation=cv2.INTER_LINEAR)
# save negative images and write label
if np.max(Iou) < 0.3:
# Iou with all gts must below 0.3
continue
else:
# find gt_box with the highest iou
idx = np.argmax(Iou)
assigned_gt = gts[idx]
x1, y1, x2, y2 = assigned_gt
# compute bbox reg label
offset_x1 = (x1 - x_left) / float(width)
offset_y1 = (y1 - y_top) / float(height)
offset_x2 = (x2 - x_right) / float(width)
offset_y2 = (y2 - y_bottom) / float(height)
offset_left_eye_x = (landmark[0, 0] - x_left) / float(width)
offset_left_eye_y = (landmark[0, 1] - y_top) / float(height)
offset_right_eye_x = (landmark[0, 2] - x_left) / float(width)
offset_right_eye_y = (landmark[0, 3] - y_top) / float(height)
offset_nose_x = (landmark[0, 4] - x_left) / float(width)
offset_nose_y = (landmark[0, 5] - y_top) / float(height)
offset_left_mouth_x = (landmark[0, 6] - x_left) / float(width)
offset_left_mouth_y = (landmark[0, 7] - y_top) / float(height)
offset_right_mouth_x = (landmark[0, 8] - x_left) / float(width)
offset_right_mouth_y = (landmark[0, 9] - y_top) / float(height)
# save positive and part-face images and write labels
if np.max(Iou) >= 0.65:
save_file = os.path.join(landmark_save_dir,
"%s.jpg" % p_idx)
f.write(save_file + ' -2 %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f \n' % \
(offset_x1, offset_y1, offset_x2, offset_y2, \
offset_left_eye_x, offset_left_eye_y, offset_right_eye_x, offset_right_eye_y,
offset_nose_x, offset_nose_y, offset_left_mouth_x, offset_left_mouth_y,
offset_right_mouth_x, offset_right_mouth_y))
cv2.imwrite(save_file, resized_im)
p_idx += 1
f.close()
