def __call__(self, data, context: dict = None): return { 'filename': change_ext(data['filename'], None, self.output_ext), 'content': doc_to_type(**self.get_doc_to_type_kwargs(data, context=context)), 'type': self.output_mime_type }
def __call__(self, data): response = requests_post(self._api_url, headers={'Content-Type': data['type']}, data=data['content']) response.raise_for_status() return { 'filename': change_ext(data['filename'], None, '.xml'), 'content': response.text, 'type': response.headers['Content-Type'] }
def process_request(self, data: dict, session: requests.Session, context: dict = None): response = self.post_data( data=data, session=session, timeout=self.get_default_request_timeout(context=context) ) return { 'filename': change_ext(data['filename'], None, '.xml'), 'content': response.text, 'type': response.headers['Content-Type'] }
def _do_convert(self, temp_source_filename, output_type: str = 'pdf', remove_line_no: bool = True, remove_header_footer: bool = True, remove_redline: bool = True): self.start_listener_if_not_running() temp_target_filename = change_ext(temp_source_filename, None, '-output.%s' % output_type) args = [] args.extend(['convert', '--format', output_type]) if remove_line_no: args.append('--remove-line-no') if remove_header_footer: args.append('--remove-header-footer') if remove_redline: args.append('--remove-redline') args.extend([ '--port', str(self.port), '--output-file', str(temp_target_filename), temp_source_filename ]) if self.no_launch: args.append('--no-launch') if self.keep_listener_running: args.append('--keep-listener-running') try: _exec_doc_converter(args, enable_debug=self.enable_debug, process_timeout=self.process_timeout) except UnoConnectionError: self.stop_listener_if_running() raise except Exception: if self.stop_listener_on_error: self.stop_listener_if_running() raise if not os.path.exists(temp_target_filename): raise RuntimeError('temp target file missing: %s' % temp_target_filename) return temp_target_filename
def __call__(self, data): return { 'filename': change_ext(data['filename'], None, '.docx'), 'content': doc_to_docx(data['content'], data['type']), 'type': MimeTypes.DOCX }
def test_should_remove_gz_ext_before_replacing_ext(self): assert change_ext('file.pdf.gz', None, '.svg.zip') == 'file.svg.zip'
def test_should_replace_simple_ext_with_combined_ext(self): assert change_ext('file.pdf', None, '.svg.zip') == 'file.svg.zip'
def test_should_replace_simple_ext_with_simple_ext(self): assert change_ext('file.pdf', None, '.xml') == 'file.xml'
def configure_pipeline(p, opt): image_size = ((opt.image_width, opt.image_height) if opt.image_width and opt.image_height else None) page_range = opt.pages first_page = page_range[0] if page_range else 1 xml_mapping = parse_xml_mapping(opt.xml_mapping_path) if opt.lxml_path: lxml_xml_file_pairs = ( p | beam.Create( [[ join_if_relative_path(opt.base_data_path, s) for s in [opt.lxml_path, opt.xml_path] ]]) | "FindFilePairs" >> TransformAndLog( beam.FlatMap(lambda patterns: islice( find_file_pairs_grouped_by_parent_directory_or_name( patterns), opt.limit)), log_prefix='file pairs: ', log_level='debug') | PreventFusion() | "ReadFileContent" >> beam.Map( lambda filenames: { 'source_filename': filenames[0], 'xml_filename': filenames[1], 'lxml_content': read_all_from_path(filenames[0]), 'xml_content': read_all_from_path(filenames[1]) })) elif opt.pdf_path or opt.pdf_xml_file_list: if opt.pdf_xml_file_list: pdf_xml_url_pairs = ( p | "ReadFilePairUrls" >> ReadDictCsv(opt.pdf_xml_file_list, limit=opt.limit) | "TranslateFilePairUrls" >> beam.Map(lambda row: (row['source_url'], row['xml_url']))) else: pdf_xml_url_pairs = (p | beam.Create([[ join_if_relative_path(opt.base_data_path, s) for s in [opt.pdf_path, opt.xml_path] ]]) | "FindFilePairs" >> TransformAndLog( beam.FlatMap(lambda patterns: islice( find_file_pairs_grouped_by_parent_directory_or_name( patterns), opt.limit)), log_prefix='file pairs: ', log_level='debug')) pdf_xml_file_pairs = ( pdf_xml_url_pairs | PreventFusion() | "ReadFileContent" >> TransformAndCount( beam.Map( lambda filenames: { 'source_filename': filenames[0], 'xml_filename': filenames[1], 'pdf_content': read_all_from_path(filenames[0]), 'xml_content': read_all_from_path(filenames[1]) }), MetricCounters.FILE_PAIR)) lxml_xml_file_pairs = ( pdf_xml_file_pairs | "ConvertPdfToLxml" >> MapOrLog( lambda v: remove_keys_from_dict( extend_dict( v, { 'lxml_content': convert_pdf_bytes_to_lxml(v['pdf_content'], path=v['source_filename' ], page_range=page_range) }), # we don't need the pdf_content unless we are writing tf_records None if opt.save_tfrecords else {'pdf_content'}), log_fn=lambda e, v: (get_logger().warning( 'caught exception (ignoring item): %s, pdf: %s, xml: %s', e, v['source_filename'], v['xml_filename'], exc_info=e)), error_count=MetricCounters.CONVERT_PDF_TO_LXML_ERROR)) else: raise RuntimeError('either lxml-path or pdf-path required') if opt.save_png or opt.save_tfrecords: with_pdf_png_pages = ( (lxml_xml_file_pairs if opt.save_tfrecords else pdf_xml_file_pairs) | "ConvertPdfToPng" >> MapOrLog( lambda v: remove_keys_from_dict( extend_dict( v, { 'pdf_png_pages': list( pdf_bytes_to_png_pages(v['pdf_content'], dpi=opt.png_dpi, image_size=image_size, page_range=page_range)) }), {'pdf_content'} # we no longer need the pdf_content ), error_count=MetricCounters.CONVERT_PDF_TO_PNG_ERROR)) if opt.save_png: _ = (with_pdf_png_pages | "SavePdfToPng" >> TransformAndLog( beam.Map(lambda v: save_pages( FileSystems.join( opt.output_path, change_ext( relative_path(opt.base_data_path, v[ 'source_filename']), None, '.png.zip')), '.png', v['pdf_png_pages'])), log_fn=lambda x: get_logger().info('saved result: %s', x))) if opt.save_lxml: _ = (lxml_xml_file_pairs | "SaveLxml" >> TransformAndLog( beam.Map(lambda v: save_file_content( FileSystems.join( opt.output_path, change_ext( relative_path(opt.base_data_path, v[ 'source_filename']), None, '.lxml.gz')), v[ 'lxml_content'])), log_fn=lambda x: get_logger().info('saved lxml: %s', x))) annotation_results = (( with_pdf_png_pages if opt.save_tfrecords else lxml_xml_file_pairs ) | "ConvertLxmlToSvgAndAnnotate" >> TransformAndCount( MapOrLog( lambda v: remove_keys_from_dict( extend_dict( v, { 'svg_pages': list( convert_and_annotate_lxml_content( v['lxml_content'], v['xml_content'], xml_mapping, name=v['source_filename'])) }), # Won't need the XML anymore {'lxml_content', 'xml_content'}), log_fn=lambda e, v: (get_logger().warning( 'caught exception (ignoring item): %s, source: %s, xml: %s', e, v['source_filename'], v['xml_filename'], exc_info=e)), error_count=MetricCounters.CONVERT_LXML_TO_SVG_ANNOT_ERROR), MetricCounters.PAGE, lambda v: len(v['svg_pages']))) if opt.save_svg: _ = (annotation_results | "SaveSvgPages" >> TransformAndLog( beam.Map(lambda v: save_svg_roots( FileSystems.join( opt.output_path, change_ext( relative_path(opt.base_data_path, v['source_filename'] ), None, '.svg.zip')), v['svg_pages'])), log_fn=lambda x: get_logger().info('saved result: %s', x))) if opt.annotation_evaluation_csv or opt.min_annotation_percentage: annotation_evaluation_results = ( annotation_results | "EvaluateAnnotations" >> TransformAndLog( beam.Map(lambda v: remove_keys_from_dict( extend_dict( v, { 'annotation_evaluation': evaluate_document_by_page( SvgStructuredDocument(v['svg_pages'])) }), None if opt.min_annotation_percentage else {'svg_pages'})), log_fn=lambda x: get_logger().info( 'annotation evaluation result: %s: %s', x[ 'source_filename'], x['annotation_evaluation']))) if opt.save_block_png or opt.save_tfrecords: color_map = parse_color_map_from_file(opt.color_map) with_block_png_pages = ( (annotation_evaluation_results if opt.min_annotation_percentage else annotation_results) | "GenerateBlockPng" >> beam.Map(lambda v: remove_keys_from_dict( extend_dict( v, { 'block_png_pages': [ svg_page_to_blockified_png_bytes( svg_page, color_map, image_size=image_size) for svg_page in v['svg_pages'] ] }), {'svg_pages'}))) if opt.save_block_png: _ = (with_block_png_pages | "SaveBlockPng" >> TransformAndLog( beam.Map(lambda v: save_pages( FileSystems.join( opt.output_path, change_ext( relative_path(opt.base_data_path, v[ 'source_filename']), None, '.block-png.zip')), '.png', v['block_png_pages'])), log_fn=lambda x: get_logger().info('saved result: %s', x))) if opt.save_tfrecords: if opt.min_annotation_percentage: filtered_pages = ( with_block_png_pages | "FilterPages" >> TransformAndCount( beam.Map(lambda v: filter_list_props_by_indices( v, get_page_indices_with_min_annotation_percentage( v['annotation_evaluation'], opt. min_annotation_percentage), {'pdf_png_pages', 'block_png_pages'})), MetricCounters.FILTERED_PAGE, lambda v: len(v['block_png_pages']))) else: filtered_pages = with_block_png_pages _ = (filtered_pages | "WriteTFRecords" >> WritePropsToTFRecord( FileSystems.join(opt.output_path, 'data'), lambda v: ({ 'input_uri': v['source_filename'] + '#page%d' % (first_page + i), 'input_image': pdf_png_page, 'annotation_uri': (v['source_filename'] + '.annot' + '#page%d' % (first_page + i)), 'annotation_image': block_png_page, 'page_no': first_page + i } for i, pdf_png_page, block_png_page in zip( range(len(v['pdf_png_pages'])), v['pdf_png_pages'], v[ 'block_png_pages'])))) if opt.annotation_evaluation_csv: annotation_evaluation_csv_name, annotation_evaluation_ext = ( os.path.splitext(opt.annotation_evaluation_csv)) _ = ( # flake8: noqa annotation_evaluation_results | "FlattenAnotationEvaluationResults" >> beam.FlatMap(lambda v: to_annotation_evaluation_csv_dict_rows( v['annotation_evaluation'], document=basename(v['source_filename']))) | "WriteAnnotationEvaluationToCsv" >> WriteDictCsv( join_if_relative_path(opt.output_path, annotation_evaluation_csv_name), file_name_suffix=annotation_evaluation_ext, columns=DEFAULT_EVALUATION_COLUMNS))
def __call__(self, data): return { 'filename': change_ext(data['filename'], None, '.pdf'), 'content': doc_to_pdf(data['content'], data['type']), 'type': MimeTypes.PDF }