def process(self, doc: TextDocument) -> bool: text_blocks = doc.text_blocks if len(text_blocks) < 2: return False # WARNING: POSSIBLE BUG FOUND: shouldn't prev_block be reset every passthrough? changes = False # if it has been changed on the previous passthrough changed_on_pass = True while changed_on_pass: changed_on_pass = False prev_block = text_blocks[0] blocks_to_remove = [] for block in text_blocks[1:]: if prev_block.is_content and block.link_density < 0.56 \ and not block.has_label(DefaultLabels.STRICTLY_NOT_CONTENT): prev_block.merge_next(block) blocks_to_remove.append(block) changed_on_pass = True changes = True else: prev_block = block text_blocks = self.subtract_blocks(text_blocks, blocks_to_remove) if changes: doc.text_blocks = text_blocks return changes
def process(self, doc: TextDocument) -> bool: text_blocks = doc.text_blocks new_blocks = [tb for tb in text_blocks if tb.is_content] has_changes = len(new_blocks) < len(text_blocks) doc.text_blocks = new_blocks return has_changes
def process(self, doc: TextDocument) -> bool: text_blocks = doc.text_blocks if len(text_blocks) < 2: return False changes = False if self.content_only: start_idx = None for idx, block in enumerate(text_blocks): if block.is_content: start_idx = idx break if start_idx is None: return False else: start_idx = 0 prev_block = text_blocks[start_idx] blocks_to_remove = [] for block in text_blocks[start_idx + 1:]: if not block.is_content: prev_block = block continue diff_blocks = (block.offset_blocks_start - prev_block.offset_blocks_end - 1) if diff_blocks <= self.max_blocks_distance: ok = True if self.content_only: if not prev_block.is_content or not block.is_content: ok = False if (self.same_tag_level_only and prev_block.tag_level != block.tag_level): ok = False if ok: prev_block.merge_next(block) # remove current block blocks_to_remove.append(block) changes = True else: prev_block = block else: prev_block = block if len(blocks_to_remove) > 0: doc.text_blocks = self.subtract_blocks(text_blocks, blocks_to_remove) changes = True return changes
def process(self, doc: TextDocument) -> bool: text_blocks = doc.text_blocks if len(text_blocks) < 2: return False changes = False prev_block = text_blocks[0] blocks_to_remove = [] for block in text_blocks[1::]: if self.equal_labels(prev_block.labels, block.labels): prev_block.merge_next(block) blocks_to_remove.append(block) changes = True else: prev_block = block if changes: doc.text_blocks = self.subtract_blocks(text_blocks, blocks_to_remove) return changes
def process(self, doc: TextDocument) -> bool: changes = False blocks = doc.text_blocks blocks_new = [] for tb in blocks: text = tb.text paragraphs = self.NEWLINE_REGEX.split(text) if len(paragraphs) < 2: blocks_new.append(tb) continue is_content = tb.is_content labels = tb.labels for p in paragraphs: tb_p = TextBlock(p) tb_p.is_content = is_content tb_p.add_labels(labels) blocks_new.append(tb_p) changes = True if changes: doc.text_blocks = blocks_new return changes