def __detect_model(self, model, image, cboxes, thres_c, thres_i, size, nms_mode='union'): """detection of Refinement Network and Output Network. """ bboxes = utils.convert_to_square(cboxes) img_data = self.__crop_boxes(image, bboxes, size) img_data = img_data.to(self.__device) _cls, _offset, _landmark = model(img_data) cls, offset, landmark = _cls[:, 0].cpu().data, _offset.cpu( ).data, _landmark.cpu().data mask = torch.gt(cls, thres_c) confidences, loffset, boffset = cls[mask], landmark[mask], offset[mask] if not any(mask): return confidences, loffset, boffset boxes = utils.cord_regression(bboxes[mask], boffset) landmarks = utils.cord_regression(bboxes[mask], loffset) keep = utils.nms(boxes, confidences, threshold=thres_i, mode=nms_mode) return confidences[keep], boxes[keep], landmarks[keep]
def gen_onet_sample_data(data_dir, anno_file, det_boxs_file, prefix): neg_save_dir = os.path.join(data_dir, "48/negative") pos_save_dir = os.path.join(data_dir, "48/positive") part_save_dir = os.path.join(data_dir, "48/part") for dir_path in [neg_save_dir, pos_save_dir, part_save_dir]: if not os.path.exists(dir_path): os.makedirs(dir_path) # load ground truth from annotation file # format of each line: image/path [x1,y1,x2,y2] for each gt_box in this image with open(anno_file, 'r') as f: annotations = f.readlines() image_size = 48 net = "onet" im_idx_list = list() gt_boxes_list = list() num_of_images = len(annotations) print("processing %d images in total" % num_of_images) for annotation in annotations: annotation = annotation.strip().split(' ') im_idx = os.path.join(prefix, annotation[0]) boxes = list(map(float, annotation[1:])) boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4) im_idx_list.append(im_idx) gt_boxes_list.append(boxes) save_path = './anno_store' if not os.path.exists(save_path): os.makedirs(save_path) f1 = open(os.path.join(save_path, 'pos_%d.txt' % image_size), 'w') f2 = open(os.path.join(save_path, 'neg_%d.txt' % image_size), 'w') f3 = open(os.path.join(save_path, 'part_%d.txt' % image_size), 'w') det_handle = open(det_boxs_file, 'rb') det_boxes = cPickle.load(det_handle) print(len(det_boxes), num_of_images) # assert len(det_boxes) == num_of_images, "incorrect detections or ground truths" # index of neg, pos and part face, used as their image names n_idx = 0 p_idx = 0 d_idx = 0 image_done = 0 for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list): if image_done % 100 == 0: print("%d images done" % image_done) image_done += 1 if dets.shape[0] == 0: continue img = cv2.imread(im_idx) dets = convert_to_square(dets) dets[:, 0:4] = np.round(dets[:, 0:4]) for box in dets: x_left, y_top, x_right, y_bottom = box[0:4].astype(int) width = x_right - x_left + 1 height = y_bottom - y_top + 1 # ignore box that is too small or beyond image border if width < 20 or x_left < 0 or y_top < 0 or x_right > img.shape[ 1] - 1 or y_bottom > img.shape[0] - 1: continue # compute intersection over union(IoU) between current box and all gt boxes Iou = IoU(box, gts) cropped_im = img[y_top:y_bottom + 1, x_left:x_right + 1, :] resized_im = cv2.resize(cropped_im, (image_size, image_size), interpolation=cv2.INTER_LINEAR) # save negative images and write label if np.max(Iou) < 0.3: # Iou with all gts must below 0.3 save_file = os.path.join(neg_save_dir, "%s.jpg" % n_idx) f2.write(save_file + ' 0\n') cv2.imwrite(save_file, resized_im) n_idx += 1 else: # find gt_box with the highest iou idx = np.argmax(Iou) assigned_gt = gts[idx] x1, y1, x2, y2 = assigned_gt # compute bbox reg label offset_x1 = (x1 - x_left) / float(width) offset_y1 = (y1 - y_top) / float(height) offset_x2 = (x2 - x_right) / float(width) offset_y2 = (y2 - y_bottom) / float(height) # save positive and part-face images and write labels if np.max(Iou) >= 0.65: save_file = os.path.join(pos_save_dir, "%s.jpg" % p_idx) f1.write(save_file + ' 1 %.2f %.2f %.2f %.2f\n' % (offset_x1, offset_y1, offset_x2, offset_y2)) cv2.imwrite(save_file, resized_im) p_idx += 1 elif np.max(Iou) >= 0.4: save_file = os.path.join(part_save_dir, "%s.jpg" % d_idx) f3.write(save_file + ' -1 %.2f %.2f %.2f %.2f\n' % (offset_x1, offset_y1, offset_x2, offset_y2)) cv2.imwrite(save_file, resized_im) d_idx += 1 f1.close() f2.close() f3.close()
def gen_rnet_sample_data(data_dir, anno_dir, det_boxs_file): ''' Generate the train data for RNet ''' part_save_dir = os.path.join(data_dir, "5keypoints/rnet/part") pos_save_dir = os.path.join(data_dir, "5keypoints/rnet/positive") neg_save_dir = os.path.join(data_dir, "5keypoints/rnet/negative") for dir_path in [neg_save_dir, pos_save_dir, part_save_dir]: if not os.path.exists(dir_path): os.makedirs(dir_path) # load ground truth from annotation file # format of each line: image/path [x1,y1,x2,y2] for each gt_box in this image anno_file = os.path.join(anno_dir, 'anno_store/wide_anno_train.txt' ) # TODO :: [local_wide_anno, wide_anno_train] with open(anno_file, 'r') as f: annotations = f.readlines() net, image_size = "rnet", 24 num_of_images = len(annotations) im_idx_list, gt_boxes_list = [], [] print("processing %d images in total" % num_of_images) for annotation in annotations: annotation = annotation.strip().split(' ') im_idx = os.path.join('', annotation[0]) # im_idx = annotation[0] boxes = list(map(float, annotation[1:])) boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4) im_idx_list.append(im_idx) gt_boxes_list.append(boxes) save_path = os.path.join(anno_dir, 'anno_store/rnet') if not os.path.exists(save_path): os.makedirs(save_path) f1 = open(os.path.join(save_path, 'pos_%d.txt' % image_size), 'w') f2 = open(os.path.join(save_path, 'neg_%d.txt' % image_size), 'w') f3 = open(os.path.join(save_path, 'part_%d.txt' % image_size), 'w') # print(det_boxs_file) det_handle = open(det_boxs_file, 'rb') det_boxes = cPickle.load(det_handle) print(len(det_boxes), num_of_images) n_idx, p_idx, d_idx = 0, 0, 0 image_done = 0 for im_idx, dets, gts in zip(im_idx_list, det_boxes, gt_boxes_list): gts = np.array(gts, dtype=np.float32).reshape(-1, 4) if image_done % 100 == 0: print("%d images done" % image_done) image_done += 1 if dets.shape[0] == 0: continue img = cv2.imread(im_idx) # change to square dets = convert_to_square(dets) dets[:, 0:4] = np.round(dets[:, 0:4]) neg_num = 0 for box in dets: x_left, y_top, x_right, y_bottom, _ = box.astype(int) width = x_right - x_left + 1 height = y_bottom - y_top + 1 # ignore box that is too small or beyond image border if width < 20 or x_left < 0 or y_top < 0 or \ (x_right > img.shape[1] - 1) or (y_bottom > img.shape[0] - 1): continue # compute intersection over union(IoU) between current box and all gt boxes Iou = IoU(box, gts) cropped_im = img[y_top:y_bottom + 1, x_left:x_right + 1, :] resized_im = cv2.resize(cropped_im, (image_size, image_size), interpolation=cv2.INTER_LINEAR) # save negative images and write label # Iou with all gts must below 0.3 if np.max(Iou) < 0.3 and neg_num < 60: # save the examples save_file = os.path.join(neg_save_dir, "%s.jpg" % n_idx) # print(save_file) f2.write(save_file + ' 0\n') cv2.imwrite(save_file, resized_im) n_idx += 1 neg_num += 1 else: # find gt_box with the highest iou idx = np.argmax(Iou) assigned_gt = gts[idx] x1, y1, x2, y2 = assigned_gt # compute bbox reg label offset_x1 = (x1 - x_left) / float(width) offset_y1 = (y1 - y_top) / float(height) offset_x2 = (x2 - x_right) / float(width) offset_y2 = (y2 - y_bottom) / float(height) # save positive and part-face images and write labels if np.max(Iou) >= 0.65: save_file = os.path.join(pos_save_dir, "%s.jpg" % p_idx) f1.write(save_file + ' 1 %.2f %.2f %.2f %.2f\n' % (offset_x1, offset_y1, offset_x2, offset_y2)) cv2.imwrite(save_file, resized_im) p_idx += 1 elif np.max(Iou) >= 0.4: save_file = os.path.join(part_save_dir, "%s.jpg" % d_idx) f3.write(save_file + ' -1 %.2f %.2f %.2f %.2f\n' % (offset_x1, offset_y1, offset_x2, offset_y2)) cv2.imwrite(save_file, resized_im) d_idx += 1 f1.close() f2.close() f3.close()
def gen_sample_data(data_dir, anno_file, det_boxs_file, prefix_path=''): landmark_save_dir = os.path.join(data_dir, "48/landmark") if not os.path.exists(landmark_save_dir): os.makedirs(landmark_save_dir) # load ground truth from annotation file # format of each line: image/path [x1,y1,x2,y2] for each gt_box in this image with open(anno_file, 'r') as f: annotations = f.readlines() image_size = 48 net = "onet" im_idx_list = list() gt_boxes_list = list() gt_landmark_list = list() num_of_images = len(annotations) print("processing %d images in total" % num_of_images) for annotation in annotations: annotation = annotation.strip().split(' ') im_idx = annotation[0] boxes = map(float, annotation[1:5]) boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4) landmarks = map(float, annotation[5:]) landmarks = np.array(landmarks, dtype=np.float32).reshape(-1, 10) im_idx_list.append(im_idx) gt_boxes_list.append(boxes) gt_landmark_list.append(landmarks) save_path = config.ANNO_STORE_DIR if not os.path.exists(save_path): os.makedirs(save_path) f = open(os.path.join(save_path, 'landmark_48.txt'), 'w') det_handle = open(det_boxs_file, 'r') det_boxes = cPickle.load(det_handle) print(len(det_boxes), num_of_images) assert len( det_boxes) == num_of_images, "incorrect detections or ground truths" # index of neg, pos and part face, used as their image names p_idx = 0 image_done = 0 for im_idx, dets, gts, landmark in zip(im_idx_list, det_boxes, gt_boxes_list, gt_landmark_list): if image_done % 100 == 0: print("%d images done" % image_done) image_done += 1 if dets.shape[0] == 0: continue img = cv2.imread(os.path.join(prefix_path, im_idx)) dets = convert_to_square(dets) dets[:, 0:4] = np.round(dets[:, 0:4]) for box in dets: x_left, y_top, x_right, y_bottom = box[0:4].astype(int) width = x_right - x_left + 1 height = y_bottom - y_top + 1 # ignore box that is too small or beyond image border if width < 20 or x_left < 0 or y_top < 0 or x_right > img.shape[ 1] - 1 or y_bottom > img.shape[0] - 1: continue # compute intersection over union(IoU) between current box and all gt boxes Iou = IoU(box, gts) cropped_im = img[y_top:y_bottom + 1, x_left:x_right + 1, :] resized_im = cv2.resize(cropped_im, (image_size, image_size), interpolation=cv2.INTER_LINEAR) # save negative images and write label if np.max(Iou) < 0.3: # Iou with all gts must below 0.3 continue else: # find gt_box with the highest iou idx = np.argmax(Iou) assigned_gt = gts[idx] x1, y1, x2, y2 = assigned_gt # compute bbox reg label offset_x1 = (x1 - x_left) / float(width) offset_y1 = (y1 - y_top) / float(height) offset_x2 = (x2 - x_right) / float(width) offset_y2 = (y2 - y_bottom) / float(height) offset_left_eye_x = (landmark[0, 0] - x_left) / float(width) offset_left_eye_y = (landmark[0, 1] - y_top) / float(height) offset_right_eye_x = (landmark[0, 2] - x_left) / float(width) offset_right_eye_y = (landmark[0, 3] - y_top) / float(height) offset_nose_x = (landmark[0, 4] - x_left) / float(width) offset_nose_y = (landmark[0, 5] - y_top) / float(height) offset_left_mouth_x = (landmark[0, 6] - x_left) / float(width) offset_left_mouth_y = (landmark[0, 7] - y_top) / float(height) offset_right_mouth_x = (landmark[0, 8] - x_left) / float(width) offset_right_mouth_y = (landmark[0, 9] - y_top) / float(height) # save positive and part-face images and write labels if np.max(Iou) >= 0.65: save_file = os.path.join(landmark_save_dir, "%s.jpg" % p_idx) f.write(save_file + ' -2 %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f %.2f \n' % \ (offset_x1, offset_y1, offset_x2, offset_y2, \ offset_left_eye_x, offset_left_eye_y, offset_right_eye_x, offset_right_eye_y, offset_nose_x, offset_nose_y, offset_left_mouth_x, offset_left_mouth_y, offset_right_mouth_x, offset_right_mouth_y)) cv2.imwrite(save_file, resized_im) p_idx += 1 f.close()