def test_make_text_content_bold_and_italic_overlapping_extend(self):
text_raw = 'Lorem Ipsum is simply'
emphasis = ['b', 'b', 'b', 'b', 'b', 'b', 'bi', 'bi', 'bi', 'bi', 'bi', 'i', 'i', 'i', 'n', 'n', 'n', 'n', 'n',
'n', 'n']
text_box = TextBox(0, 0, 0, 0, Text(text_raw, '', 0, emphasis))
text = make_text_content(text_box)
self.assertEqual(text, '<b>Lorem <i>Ipsum</i></b><i> is</i> simply', 'Wrong formatting')
def _parse_layout_objects(self, layout, page: Page, image_path):
"""Iterate through the list of LT* objects and capture the text or image data contained in each"""
page_text = {
} # k=(x0, x1) of the bbox, v=list of text strings within that bbox width (physical column)
for lt_obj in layout:
if isinstance(lt_obj, LTTextBox) or isinstance(
lt_obj, LTTextLine) or isinstance(lt_obj,
LTTextLineHorizontal):
self._get_text(page_text, lt_obj)
elif isinstance(lt_obj, LTImage):
image_name, file_stream = save_image(
lt_obj, 'page_{}_{}_{}'.format(page.number, lt_obj.y0,
lt_obj.x0), image_path)
if image_name:
# top = page height - image height - y0, i.e. y0 is the distance from the bottom of the page
top = page.height - round(
lt_obj.y0) - round(lt_obj.y1 - lt_obj.y0)
left = round(lt_obj.x0)
width = min(page.width, round(lt_obj.x1 - lt_obj.x0))
height = round(lt_obj.y1 - lt_obj.y0)
page.add_box(
ImageBox(top, left, width, height, image_name,
file_stream))
elif isinstance(lt_obj, LTFigure):
# LTFigure objects are containers for other LT* objects, so recurse through the children
self._parse_layout_objects(lt_obj, page, image_path)
page_text = [(k[0], k[1], k, v) for k, v in page_text.items()]
page_text = list(
sorted(sorted(page_text, key=itemgetter(0)),
key=itemgetter(1),
reverse=True))
previous_top, aligned_text = 0, []
for _, _, k, v in page_text:
top = k[1]
if abs(ceil(previous_top) - ceil(top)) <= 2:
top = previous_top
# Skip empty text in the beginning of line. Usually these are empty lines
if top > previous_top and v.text.isspace():
continue
previous_top = top
aligned_text.append((k[0], int(top), (k[0], top, k[2], k[3]), v))
aligned_text = list(
sorted(sorted(aligned_text, key=itemgetter(0)),
key=itemgetter(1),
reverse=True))
for _, _, k, v in aligned_text:
page.add_box(
TextBox(int(floor(page.height - k[1])), int(floor(k[0])),
int(ceil(k[2] - k[0]) + 5), int(ceil(k[3] - k[1])), v))
def _convert_pdfxml_to_document(self, xml_filename):
if xml_filename is not None or self._intermediate_xml is None:
xml_content = []
with open(xml_filename, 'r', encoding='utf-8') as f:
links = re.compile('<a href=".*?">|<\/a>')
spans = re.compile('<span( class=".*?")*>|<\/span>')
multi_spaces = re.compile('( )+')
quotes = re.compile('�')
control_chars = re.compile(r'[\x00-\x1f\x7f-\x9f]')
for line in f:
line = control_chars.sub('', line)
line = links.sub('', spans.sub('', line))
line = multi_spaces.sub(' ', quotes.sub('"', line))
xml_content.append(line)
self._intermediate_xml = ''.join(xml_content)
dom_tree = xml.dom.minidom.parseString(self._intermediate_xml)
self._document, fonts = Document(), {}
for page in dom_tree.documentElement.getElementsByTagName('page'):
current_page = Page(number=int(page.getAttribute('number')),
width=int(page.getAttribute('width')),
height=int(page.getAttribute('height')))
for node in [
node for node in page.childNodes
if node.nodeName == 'fontspec'
]:
font_id = int(node.getAttribute('id'))
font_node = {
'size': int(node.getAttribute('size')),
'family': node.getAttribute('family'),
'color': node.getAttribute('color')
}
fonts[font_id] = font_node
nodes = sorted([
node for node in page.childNodes
if node.nodeName == 'text' and node.nodeType == 1
],
key=lambda x: int(x.getAttribute('top')))
page_text, previous_top = {}, 0
for i, node in enumerate(nodes):
top, left = int(node.getAttribute('top')), int(
node.getAttribute('left'))
width, height = int(node.getAttribute('width')), int(
node.getAttribute('height'))
if abs(ceil(previous_top) - ceil(top)) <= floor(height / 2):
top = previous_top
previous_top = top
text = Converter._get_tag_from_xml_node(node)
if text is None or not text:
continue
if isinstance(text, list):
text = ''.join(text)
font_details = fonts[int(node.getAttribute('font'))]
content = Text(text=text,
font=font_details['family'],
size=font_details['size'])
page_text[(top, left, width, height)] = content
# sort the page_text hash by the keys (x0,x1 values of the bbox),
# which produces a top-down, left-to-right sequence of related columns
page_text_items = [(k[0], k[1], k, v)
for k, v in page_text.items()]
page_text_items = sorted(page_text_items, key=itemgetter(0, 1))
sorted_text = [(c, d) for a, b, c, d in page_text_items]
for k, v in sorted_text:
current_page.add_box(
TextBox(int(k[0]), int(k[1]), int(k[2]), int(k[3]), v))
self._document.add_page(current_page)
def test_make_text_content_no_formatting(self):
text_raw = 'Lorem Ipsum is simply'
emphasis = ['n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n']
text_box = TextBox(0, 0, 0, 0, Text(text_raw, '', 0, emphasis))
text = make_text_content(text_box)
self.assertEqual(text, text_raw, 'Wrong formatting')
def test_make_text_content_bold_middle(self):
text_raw = 'Lorem Ipsum is simply'
emphasis = ['n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'n', 'b', 'b', 'n', 'n', 'n', 'n', 'n', 'n', 'n']
text_box = TextBox(0, 0, 0, 0, Text(text_raw, '', 0, emphasis))
text = make_text_content(text_box)
self.assertEqual(text, 'Lorem Ipsum <b>is</b> simply', 'Wrong formatting')