def extract_figures_json(pdf_path, page_image_paths, pdffigures_output,
output_directory):
"""Extract information about figures to JSON and save to disk.
:param str pdf_path: path to the PDF from which to extract
figures.
:returns: path to the JSON file containing the detection results.
"""
page_images_array = np.array(
[imread(page_image_path) for page_image_path in page_image_paths])
detector = get_detector()
figure_boxes_by_page = detector.get_detections(page_images_array)
pdffigures_captions = pdffigures_wrapper.get_captions(
pdffigures_output=pdffigures_output,
target_dpi=settings.DEFAULT_INFERENCE_DPI)
figures_by_page = []
for page_num in range(len(page_image_paths)):
figure_boxes = figure_boxes_by_page[page_num]
pf_page_captions = [
caption for caption in pdffigures_captions
if caption.page == page_num
]
caption_boxes = [
caption.caption_boundary for caption in pf_page_captions
]
figure_indices, caption_indices = figure_utils.pair_boxes(
figure_boxes, caption_boxes)
page_image = page_images_array[page_num]
pad_pixels = PAD_FACTOR * min(page_image.shape[:2])
for (figure_idx, caption_idx) in zip(figure_indices, caption_indices):
figures_by_page.append(
Figure(figure_boundary=figure_boxes[figure_idx].expand_box(
pad_pixels).crop_to_page(
page_image.shape).crop_whitespace_edges(page_image),
caption_boundary=caption_boxes[caption_idx],
caption_text=pf_page_captions[caption_idx].caption_text,
name=pf_page_captions[caption_idx].name,
figure_type=pf_page_captions[caption_idx].figure_type,
page=page_num))
pdf_detection_result = PdfDetectionResult(
pdf=pdf_path,
figures=figures_by_page,
dpi=settings.DEFAULT_INFERENCE_DPI,
raw_detected_boxes=figure_boxes_by_page,
raw_pdffigures_output=pdffigures_output)
output_path = os.path.join(
output_directory,
os.path.basename(pdf_path)[:-4] + 'deepfigures-results.json')
file_util.write_json_atomic(output_path,
pdf_detection_result.to_dict(),
indent=2,
sort_keys=True)
return output_path
def detect_figures(
pdf: str, pdffigures_captions: List[CaptionOnly],
detector: TensorboxCaptionmaskDetector,
conf_threshold: float) -> Tuple[List[Figure], List[List[BoxClass]]]:
page_image_files = pdf_renderer.render(pdf,
dpi=settings.DEFAULT_INFERENCE_DPI)
page_tensors = []
for f in page_image_files:
page_im = image_util.read_tensor(f)
if detector.hypes['image_channels'] == 3:
page_tensors.append(page_im)
else:
im_with_mask = np.pad(page_im,
pad_width=[(0, 0), (0, 0), (0, 1)],
mode='constant',
constant_values=CAPTION_CHANNEL_BACKGROUND)
for caption in pdffigures_captions:
(x1, y1, x2, y2) = caption.caption_boundary.get_rounded()
im_with_mask[y1:y2, x1:x2, 3] = CAPTION_CHANNEL_MASK
page_tensors.append(im_with_mask)
figure_boxes_by_page = detector.get_detections(
page_tensors, conf_threshold=conf_threshold)
figures_by_page = []
for page_num in range(len(page_image_files)):
# Page numbers are always 0 indexed
figure_boxes = figure_boxes_by_page[page_num]
pf_page_captions = [
cap for cap in pdffigures_captions if cap.page == page_num
]
caption_boxes = [cap.caption_boundary for cap in pf_page_captions]
(figure_indices,
caption_indices) = figure_utils.pair_boxes(figure_boxes,
caption_boxes)
figures_by_page.extend([
Figure(
figure_boundary=figure_boxes[figure_idx],
caption_boundary=caption_boxes[caption_idx],
caption_text=pf_page_captions[caption_idx].caption_text,
name=pf_page_captions[caption_idx].name,
figure_type=pf_page_captions[caption_idx].figure_type,
page=page_num,
) for (figure_idx,
caption_idx) in zip(figure_indices, caption_indices)
])
return figures_by_page, figure_boxes_by_page
def evaluate_dataset_on_weights(hidden_set_dir: str, hidden_set_images_subdir: str, save_dir: str, iteration: int):
detector_args = {
'save_dir': save_dir,
'iteration': iteration
}
detector = get_detector(detector_args=detector_args)
annos = json.load(open(os.path.join(hidden_set_dir, 'figure_boundaries.json')))
for anno in annos:
image_np = imageio.imread(os.path.join(hidden_set_dir, hidden_set_images_subdir, anno['image_path']))
pred_boxes = detector.get_detections([image_np])[0]
true_boxes = [BoxClass(x1=rect['x1'], y1=rect['y1'], x2=rect['x2'], y2=rect['y2']) for rect in anno['rects']]
(pred_indices, true_indices) = figure_utils.pair_boxes(pred_boxes, true_boxes)
print("Pred boxes: ", pred_boxes)
print("True boxes: ", true_boxes)
print("Pred indices: ", pred_indices)
print("True indices: ", true_indices)
a = 0
def compute_page_ious(pred_boxes: List[BoxClass], true_boxes: List[BoxClass], iou_thresh: float) -> Tuple[
List[float], int, int, int]:
"""
Reference: https://towardsdatascience.com/evaluating-performance-of-an-object-detection-model-137a349c517b
:param pred_boxes:
:param true_boxes:
:return:
"""
(true_indices, pred_indices) = figure_utils.pair_boxes(true_boxes, pred_boxes)
tp, fp, fn = 0, 0, 0
ious = []
for (true_idx, pred_idx) in zip(true_indices, pred_indices):
iou = true_boxes[true_idx].iou(pred_boxes[pred_idx])
ious.append(iou)
if iou >= iou_thresh:
tp = tp + 1
if iou < 0.8:
fp = fp + 1
fn = len(true_boxes) - len(true_indices)
assert fn >= 0
return ious + [0.0] * (
max(len(pred_boxes), len(true_boxes)) - max(len(pred_indices), len(true_indices))), tp, fp, fn
TOTAL_IOU_FAILED_2: 0
}
for image_path in set_1:
anno_1 = figures_boundaries_1_dict[image_path]
anno_2 = figures_boundaries_2_dict[image_path]
if len(anno_1['rects']) != len(anno_2['rects']):
logger.info("Different number of boxes found.")
# Transform annos to BoxClass
boxes_1 = [rect_to_box_class(rect) for rect in anno_1['rects']]
boxes_2 = [rect_to_box_class(rect) for rect in anno_2['rects']]
# Hungarian matching
(indices_1, indices_2) = figure_utils.pair_boxes(boxes_1, boxes_2)
if len(indices_1) != len(boxes_1) or len(indices_2) != len(boxes_2):
logger.info("Unmatched boxes found.")
# Find unmatched boxes
unmatched_boxes_1 = [
box for idx, box in enumerate(boxes_1) if idx not in indices_1
]
unmatched_boxes_2 = [
box for idx, box in enumerate(boxes_2) if idx not in indices_2
]
# Initialize lists for the loop
iou_passed_boxes_1 = []
iou_passed_boxes_2 = []
if __name__ == "__main__":
path_to_figure_boundaries_with_hidden_detection_file = "/home/sampanna/workspace/bdts2/deepfigures-results/model_checkpoints/377266_arxiv_2020-06-02_22-48-45/figure_boundaries_hidden_set_501101.json"
dataset_dir = '/home/sampanna/workspace/bdts2/deepfigures-results/gold_standard_dataset'
annos = json.load(
open(path_to_figure_boundaries_with_hidden_detection_file))
counter = 0
for anno in annos:
true_boxes = [rect_to_box_class(rect) for rect in anno['rects']]
pred_boxes = [
rect_to_box_class(rect) for rect in anno['hidden_set_rects']
]
(true_indices,
pred_indices) = figure_utils.pair_boxes(true_boxes, pred_boxes)
if len(true_boxes) > 2 and len(pred_boxes) > 2:
counter = counter + 1
(fig, ax) = plot_bbs(os.path.join(dataset_dir, 'images',
anno['image_path']),
bbs=true_boxes,
bb_color='g')
(fig, ax) = plot_bbs(os.path.join(dataset_dir, 'images',
anno['image_path']),
bbs=pred_boxes,
bb_color='r',
fig=fig,
ax=ax)
plt.legend(handles=[
Patch(color='r', label='Predicted'),
Patch(color='g', label='Ground truth'),