def __init__(self, config: AppConfig): self.config = config self.download_manager = DownloadManager( download_dir=get_download_dir(config)) self.pdfalto_wrapper = PdfAltoWrapper( self.download_manager.download_if_url(config['pdfalto']['path'])) self.pdfalto_wrapper.ensure_executable() self.app_context = AppContext( app_config=config, download_manager=self.download_manager, lazy_wapiti_binary_wrapper=LazyWapitiBinaryWrapper( install_url=config.get('wapiti', {}).get('install_source'), download_manager=self.download_manager)) self.fulltext_processor_config = FullTextProcessorConfig.from_app_config( app_config=config) self.fulltext_models = load_models( config, app_context=self.app_context, fulltext_processor_config=self.fulltext_processor_config) if config.get('preload_on_startup'): self.fulltext_models.preload() self.app_features_context = load_app_features_context( config, download_manager=self.download_manager) tei_to_jats_config = config.get('xslt', {}).get('tei_to_jats', {}) self.tei_to_jats_xslt_transformer = XsltTransformerWrapper.from_template_file( TEI_TO_JATS_XSLT_FILE, xslt_template_parameters=tei_to_jats_config.get('parameters', {})) self.doc_to_pdf_enabled = config.get('doc_to_pdf', {}).get('enabled', True) self.doc_to_pdf_convert_parameters = config.get('doc_to_pdf', {}).get('convert', {}) self.doc_converter_wrapper = DocConverterWrapper( **config.get('doc_to_pdf', {}).get('listener', {}))
def load_from(model_path: str, download_manager: DownloadManager, wapiti_binary_path: str = None) -> 'WapitiModelAdapter': model_file_path = os.path.join(model_path, 'model.wapiti.gz') local_model_file_path = None try: local_model_file_path = download_manager.download_if_url( model_file_path) except FileNotFoundError: pass if not local_model_file_path or not os.path.isfile( str(local_model_file_path)): model_file_path = os.path.splitext(model_file_path)[0] local_model_file_path = download_manager.download_if_url( model_file_path) LOGGER.debug('local_model_file_path: %s', local_model_file_path) if local_model_file_path.endswith('.gz'): local_uncompressed_file_path = os.path.splitext( local_model_file_path)[0] copy_file(local_model_file_path, local_uncompressed_file_path, overwrite=False) local_model_file_path = local_uncompressed_file_path return WapitiModelAdapter( WapitiWrapper(wapiti_binary_path=wapiti_binary_path), model_file_path=local_model_file_path, model_path=model_path)
def download_if_url_from_alternatives( download_manager: DownloadManager, alternative_file_url_or_path_list: Sequence[str]) -> str: for file_url_or_path in alternative_file_url_or_path_list: if not is_external_location(file_url_or_path): if os.path.exists(file_url_or_path): return file_url_or_path LOGGER.debug('local file doesnt exist: %r', file_url_or_path) continue local_file = download_manager.get_local_file(file_url_or_path) if os.path.exists(local_file): return local_file LOGGER.debug('no existing local files found, downloading: %r', alternative_file_url_or_path_list) for file_url_or_path in alternative_file_url_or_path_list: try: local_file = download_manager.download_if_url(file_url_or_path) if os.path.exists(local_file): return local_file LOGGER.debug('local file for %r not found: %r', file_url_or_path, local_file) except FileNotFoundError: LOGGER.debug('remote file not found: %r', file_url_or_path) raise FileNotFoundError('no file found for %r' % alternative_file_url_or_path_list)
def test_should_download_using_passed_in_local_file( self, copy_file_mock: MagicMock, download_dir: Path): download_file = str(download_dir.joinpath('custom.file')) download_manager = DownloadManager(download_dir=str(download_dir)) assert download_manager.download( EXTERNAL_TXT_URL_1, local_file=download_file) == download_file copy_file_mock.assert_called_with(EXTERNAL_TXT_URL_1, download_file)
def _pdfalto_wrapper(sciencebeam_parser_config: dict) -> PdfAltoWrapper: download_manager = DownloadManager( download_dir=get_download_dir(sciencebeam_parser_config)) pdfalto_wrapper = PdfAltoWrapper( download_manager.download_if_url( sciencebeam_parser_config['pdfalto']['path'])) pdfalto_wrapper.ensure_executable() return pdfalto_wrapper
def do_run(self, args: argparse.Namespace): LOGGER.info('train') download_manager = DownloadManager() train_input_paths = _flatten_input_paths(args.train_input) train_input_texts, train_input_labels, list_classes = load_input_data( train_input_paths, download_manager=download_manager, limit=args.train_input_limit ) LOGGER.info('list_classes: %s', list_classes) embedding_name = self.preload_and_validate_embedding( args.embeddings, use_word_embeddings=True ) train( app_config=self.app_config, model_config=ModelConfig( embeddings_name=embedding_name, model_type=args.architecture, list_classes=list_classes ), training_config=TrainingConfig( batch_size=args.batch_size, max_epoch=args.max_epoch, log_dir=args.checkpoint ), train_input_texts=train_input_texts, train_input_labels=train_input_labels, model_path=args.model_path )
def get_downloaded_input_paths( input_paths: List[str], download_manager: DownloadManager) -> List[str]: return [ download_manager.download_if_url(input_path) for input_path in input_paths ]
def run(self, args: argparse.Namespace): if args.save_input_to_and_exit: save_input_to(args.input, args.save_input_to_and_exit) return self.download_manager = DownloadManager() self.embedding_manager = EmbeddingManager( download_manager=self.download_manager) if args.no_use_lmdb: self.embedding_manager.disable_embedding_lmdb_cache() set_random_seeds(args.random_seed) self.do_run(args) # see https://github.com/tensorflow/tensorflow/issues/3388 K.clear_session()
def __init__( self, download_manager: DownloadManager = None, embedding_registry_path: str = None, embedding_manager: EmbeddingManager = None, **kwargs): self.embedding_registry_path = embedding_registry_path or DEFAULT_EMBEDDINGS_PATH if download_manager is None: download_manager = DownloadManager() if embedding_manager is None: embedding_manager = EmbeddingManager( path=self.embedding_registry_path, download_manager=download_manager ) self.download_manager = download_manager self.embedding_manager = embedding_manager super().__init__(**kwargs)
def load_data_and_labels(input_paths: List[str] = None, limit: int = None, shuffle_input: bool = False, clean_features: bool = True, random_seed: int = DEFAULT_RANDOM_SEED, download_manager: DownloadManager = None): assert download_manager assert input_paths LOGGER.info('loading data from: %s', input_paths) downloaded_input_paths = [ download_manager.download_if_url(input_path) for input_path in input_paths ] x_all, y_all, f_all = _load_data_and_labels_crf_files( downloaded_input_paths, limit=limit) if shuffle_input: shuffle_arrays([x_all, y_all, f_all], random_seed=random_seed) log_data_info(x_all, y_all, f_all) if clean_features: (x_all, y_all, f_all) = get_clean_x_y_features(x_all, y_all, f_all) return x_all, y_all, f_all
def run(self, args: argparse.Namespace): self.download_manager = DownloadManager() self.embedding_manager = EmbeddingManager( download_manager=self.download_manager ) self.app_config = AppConfig( download_manager=self.download_manager, embedding_manager=self.embedding_manager ) if args.no_use_lmdb: self.embedding_manager.disable_embedding_lmdb_cache() if args.preload_embedding: self.preload_and_validate_embedding( args.preload_embedding, use_word_embeddings=True ) self.do_run(args) # see https://github.com/tensorflow/tensorflow/issues/3388 K.clear_session()
def do_run(self, args: argparse.Namespace): LOGGER.info('eval') download_manager = DownloadManager() eval_input_paths = _flatten_input_paths(args.eval_input) eval_label_input_paths = _flatten_input_paths(args.eval_label_input) eval_input_texts, eval_input_labels, list_classes = load_input_data( eval_input_paths, download_manager=download_manager, limit=args.eval_input_limit ) if eval_label_input_paths: eval_input_labels, _ = load_label_data( eval_label_input_paths, download_manager=download_manager, limit=args.eval_input_limit ) LOGGER.info('list_classes: %s', list_classes) result = evaluate( app_config=self.app_config, eval_input_texts=eval_input_texts, eval_input_labels=eval_input_labels, model_path=args.model_path ) print(result.text_formatted_report)
def do_run(self, args: argparse.Namespace): LOGGER.info('train') download_manager = DownloadManager() predict_input_paths = _flatten_input_paths(args.predict_input) predict_df = load_input_data_frame( predict_input_paths, download_manager=download_manager, limit=args.predict_input_limit ) predict_input_texts, _, _ = get_texts_and_classes_from_data_frame( predict_df ) result = predict( app_config=self.app_config, eval_input_texts=predict_input_texts, model_path=args.model_path ) list_classes = result['labels'] prediction = result['prediction'] LOGGER.info('list_classes: %s', list_classes) result_df = pd.concat([ predict_df[predict_df.columns[:2]], pd.DataFrame( prediction, columns=list_classes, index=predict_df.index ) ], axis=1) if args.predict_output: LOGGER.info('writing output to: %s', args.predict_output) save_data_frame(result_df, args.predict_output) else: print(json.dumps( result_df.to_dict(orient='records'), indent=2 ))
def install_wapiti_and_get_path_or_none( install_url: Optional[str], download_manager: DownloadManager) -> Optional[str]: if not install_url: return None if not install_url.endswith(TAR_GZ_EXT): raise ValueError('only supporting %s' % TAR_GZ_EXT) local_file = download_manager.download_if_url(install_url, auto_uncompress=False) extracted_directory = local_file[:-len(TAR_GZ_EXT)] LOGGER.debug('local_file: %s', local_file) LOGGER.debug('extracting to: %s', extracted_directory) with tarfile.open(local_file, mode='r') as tar: tar.extractall(extracted_directory) extracted_files = os.listdir(extracted_directory) wapiti_source_directory = extracted_directory if len(extracted_files) == 1: wapiti_source_directory = os.path.join(extracted_directory, extracted_files[0]) LOGGER.info('running make in %s', wapiti_source_directory) subprocess.check_output('make', cwd=wapiti_source_directory) wapiti_binary = os.path.join(wapiti_source_directory, 'wapiti') LOGGER.info('done, binary: %s', wapiti_binary) return wapiti_binary
def test_should_unzip_embedding(self, copy_file_mock: MagicMock, download_manager: DownloadManager): download_file = download_manager.download(EXTERNAL_TXT_GZ_URL_1) copy_file_mock.assert_called_with(EXTERNAL_TXT_GZ_URL_1, download_file) assert str(download_file).endswith( os.path.basename(EXTERNAL_TXT_URL_1))
def __init__(self, download_manager: DownloadManager = None): if download_manager is None: download_manager = DownloadManager() self.download_manager = download_manager
def test_get_local_file_should_return_different_path_for_different_url_paths( self, download_manager: DownloadManager): assert (download_manager.get_local_file('http://host1/file.txt') != download_manager.get_local_file('http://host2/file.txt'))
def test_get_local_file_should_return_same_path_for_same_urls( self, download_manager: DownloadManager): assert (download_manager.get_local_file(EXTERNAL_TXT_URL_1) == download_manager.get_local_file(EXTERNAL_TXT_URL_1))
def _download_manager(download_dir: Path): return DownloadManager(download_dir=str(download_dir))
def __init__( self, *args, use_features: bool = False, features_indices: List[int] = None, features_embedding_size: int = None, multiprocessing: bool = False, embedding_registry_path: str = None, embedding_manager: EmbeddingManager = None, config_props: dict = None, training_props: dict = None, max_sequence_length: int = None, input_window_stride: int = None, eval_max_sequence_length: int = None, eval_input_window_stride: int = None, batch_size: int = None, eval_batch_size: int = None, stateful: bool = None, transfer_learning_config: TransferLearningConfig = None, tag_transformed: bool = False, **kwargs): # initialise logging if not already initialised logging.basicConfig(level='INFO') LOGGER.debug('Sequence, args=%s, kwargs=%s', args, kwargs) self.embedding_registry_path = embedding_registry_path or DEFAULT_EMBEDDINGS_PATH if embedding_manager is None: embedding_manager = EmbeddingManager( path=self.embedding_registry_path, download_manager=DownloadManager() ) self.download_manager = embedding_manager.download_manager self.embedding_manager = embedding_manager self.embeddings: Optional[Embeddings] = None if not batch_size: batch_size = get_default_batch_size() if not max_sequence_length: max_sequence_length = get_default_max_sequence_length() self.max_sequence_length = max_sequence_length if not input_window_stride: input_window_stride = get_default_input_window_stride() self.input_window_stride = input_window_stride self.eval_max_sequence_length = eval_max_sequence_length self.eval_input_window_stride = eval_input_window_stride self.eval_batch_size = eval_batch_size self.model_path: Optional[str] = None if stateful is None: # use a stateful model, if supported stateful = get_default_stateful() self.stateful = stateful self.transfer_learning_config = transfer_learning_config self.dataset_transformer_factory = DummyDatasetTransformer self.tag_transformed = tag_transformed super().__init__( *args, max_sequence_length=max_sequence_length, batch_size=batch_size, **kwargs ) LOGGER.debug('use_features=%s', use_features) self.model_config: ModelConfig = ModelConfig( **{ # type: ignore **vars(self.model_config), **(config_props or {}), 'features_indices': features_indices, 'features_embedding_size': features_embedding_size }, use_features=use_features ) self.update_model_config_word_embedding_size() updated_implicit_model_config_props(self.model_config) self.update_dataset_transformer_factor() self.training_config: TrainingConfig = TrainingConfig( **vars(cast(DelftTrainingConfig, self.training_config)), **(training_props or {}) ) LOGGER.info('training_config: %s', vars(self.training_config)) self.multiprocessing = multiprocessing self.tag_debug_reporter = get_tag_debug_reporter_if_enabled() self._load_exception = None self.p: Optional[WordPreprocessor] = None self.model: Optional[BaseModel] = None self.models: List[BaseModel] = []