def extract_figures_json(pdf_path, page_image_paths, pdffigures_output, output_directory): """Extract information about figures to JSON and save to disk. :param str pdf_path: path to the PDF from which to extract figures. :returns: path to the JSON file containing the detection results. """ page_images_array = np.array( [imread(page_image_path) for page_image_path in page_image_paths]) detector = get_detector() figure_boxes_by_page = detector.get_detections(page_images_array) pdffigures_captions = pdffigures_wrapper.get_captions( pdffigures_output=pdffigures_output, target_dpi=settings.DEFAULT_INFERENCE_DPI) figures_by_page = [] for page_num in range(len(page_image_paths)): figure_boxes = figure_boxes_by_page[page_num] pf_page_captions = [ caption for caption in pdffigures_captions if caption.page == page_num ] caption_boxes = [ caption.caption_boundary for caption in pf_page_captions ] figure_indices, caption_indices = figure_utils.pair_boxes( figure_boxes, caption_boxes) page_image = page_images_array[page_num] pad_pixels = PAD_FACTOR * min(page_image.shape[:2]) for (figure_idx, caption_idx) in zip(figure_indices, caption_indices): figures_by_page.append( Figure(figure_boundary=figure_boxes[figure_idx].expand_box( pad_pixels).crop_to_page( page_image.shape).crop_whitespace_edges(page_image), caption_boundary=caption_boxes[caption_idx], caption_text=pf_page_captions[caption_idx].caption_text, name=pf_page_captions[caption_idx].name, figure_type=pf_page_captions[caption_idx].figure_type, page=page_num)) pdf_detection_result = PdfDetectionResult( pdf=pdf_path, figures=figures_by_page, dpi=settings.DEFAULT_INFERENCE_DPI, raw_detected_boxes=figure_boxes_by_page, raw_pdffigures_output=pdffigures_output) output_path = os.path.join( output_directory, os.path.basename(pdf_path)[:-4] + 'deepfigures-results.json') file_util.write_json_atomic(output_path, pdf_detection_result.to_dict(), indent=2, sort_keys=True) return output_path
def detect_figures( pdf: str, pdffigures_captions: List[CaptionOnly], detector: TensorboxCaptionmaskDetector, conf_threshold: float) -> Tuple[List[Figure], List[List[BoxClass]]]: page_image_files = pdf_renderer.render(pdf, dpi=settings.DEFAULT_INFERENCE_DPI) page_tensors = [] for f in page_image_files: page_im = image_util.read_tensor(f) if detector.hypes['image_channels'] == 3: page_tensors.append(page_im) else: im_with_mask = np.pad(page_im, pad_width=[(0, 0), (0, 0), (0, 1)], mode='constant', constant_values=CAPTION_CHANNEL_BACKGROUND) for caption in pdffigures_captions: (x1, y1, x2, y2) = caption.caption_boundary.get_rounded() im_with_mask[y1:y2, x1:x2, 3] = CAPTION_CHANNEL_MASK page_tensors.append(im_with_mask) figure_boxes_by_page = detector.get_detections( page_tensors, conf_threshold=conf_threshold) figures_by_page = [] for page_num in range(len(page_image_files)): # Page numbers are always 0 indexed figure_boxes = figure_boxes_by_page[page_num] pf_page_captions = [ cap for cap in pdffigures_captions if cap.page == page_num ] caption_boxes = [cap.caption_boundary for cap in pf_page_captions] (figure_indices, caption_indices) = figure_utils.pair_boxes(figure_boxes, caption_boxes) figures_by_page.extend([ Figure( figure_boundary=figure_boxes[figure_idx], caption_boundary=caption_boxes[caption_idx], caption_text=pf_page_captions[caption_idx].caption_text, name=pf_page_captions[caption_idx].name, figure_type=pf_page_captions[caption_idx].figure_type, page=page_num, ) for (figure_idx, caption_idx) in zip(figure_indices, caption_indices) ]) return figures_by_page, figure_boxes_by_page
def evaluate_dataset_on_weights(hidden_set_dir: str, hidden_set_images_subdir: str, save_dir: str, iteration: int): detector_args = { 'save_dir': save_dir, 'iteration': iteration } detector = get_detector(detector_args=detector_args) annos = json.load(open(os.path.join(hidden_set_dir, 'figure_boundaries.json'))) for anno in annos: image_np = imageio.imread(os.path.join(hidden_set_dir, hidden_set_images_subdir, anno['image_path'])) pred_boxes = detector.get_detections([image_np])[0] true_boxes = [BoxClass(x1=rect['x1'], y1=rect['y1'], x2=rect['x2'], y2=rect['y2']) for rect in anno['rects']] (pred_indices, true_indices) = figure_utils.pair_boxes(pred_boxes, true_boxes) print("Pred boxes: ", pred_boxes) print("True boxes: ", true_boxes) print("Pred indices: ", pred_indices) print("True indices: ", true_indices) a = 0
def compute_page_ious(pred_boxes: List[BoxClass], true_boxes: List[BoxClass], iou_thresh: float) -> Tuple[ List[float], int, int, int]: """ Reference: https://towardsdatascience.com/evaluating-performance-of-an-object-detection-model-137a349c517b :param pred_boxes: :param true_boxes: :return: """ (true_indices, pred_indices) = figure_utils.pair_boxes(true_boxes, pred_boxes) tp, fp, fn = 0, 0, 0 ious = [] for (true_idx, pred_idx) in zip(true_indices, pred_indices): iou = true_boxes[true_idx].iou(pred_boxes[pred_idx]) ious.append(iou) if iou >= iou_thresh: tp = tp + 1 if iou < 0.8: fp = fp + 1 fn = len(true_boxes) - len(true_indices) assert fn >= 0 return ious + [0.0] * ( max(len(pred_boxes), len(true_boxes)) - max(len(pred_indices), len(true_indices))), tp, fp, fn
TOTAL_IOU_FAILED_2: 0 } for image_path in set_1: anno_1 = figures_boundaries_1_dict[image_path] anno_2 = figures_boundaries_2_dict[image_path] if len(anno_1['rects']) != len(anno_2['rects']): logger.info("Different number of boxes found.") # Transform annos to BoxClass boxes_1 = [rect_to_box_class(rect) for rect in anno_1['rects']] boxes_2 = [rect_to_box_class(rect) for rect in anno_2['rects']] # Hungarian matching (indices_1, indices_2) = figure_utils.pair_boxes(boxes_1, boxes_2) if len(indices_1) != len(boxes_1) or len(indices_2) != len(boxes_2): logger.info("Unmatched boxes found.") # Find unmatched boxes unmatched_boxes_1 = [ box for idx, box in enumerate(boxes_1) if idx not in indices_1 ] unmatched_boxes_2 = [ box for idx, box in enumerate(boxes_2) if idx not in indices_2 ] # Initialize lists for the loop iou_passed_boxes_1 = [] iou_passed_boxes_2 = []
if __name__ == "__main__": path_to_figure_boundaries_with_hidden_detection_file = "/home/sampanna/workspace/bdts2/deepfigures-results/model_checkpoints/377266_arxiv_2020-06-02_22-48-45/figure_boundaries_hidden_set_501101.json" dataset_dir = '/home/sampanna/workspace/bdts2/deepfigures-results/gold_standard_dataset' annos = json.load( open(path_to_figure_boundaries_with_hidden_detection_file)) counter = 0 for anno in annos: true_boxes = [rect_to_box_class(rect) for rect in anno['rects']] pred_boxes = [ rect_to_box_class(rect) for rect in anno['hidden_set_rects'] ] (true_indices, pred_indices) = figure_utils.pair_boxes(true_boxes, pred_boxes) if len(true_boxes) > 2 and len(pred_boxes) > 2: counter = counter + 1 (fig, ax) = plot_bbs(os.path.join(dataset_dir, 'images', anno['image_path']), bbs=true_boxes, bb_color='g') (fig, ax) = plot_bbs(os.path.join(dataset_dir, 'images', anno['image_path']), bbs=pred_boxes, bb_color='r', fig=fig, ax=ax) plt.legend(handles=[ Patch(color='r', label='Predicted'), Patch(color='g', label='Ground truth'),