def document_single_builder_fn(tmp_path): yield functools.partial( document_datasets._document_single_builder, visu_doc_util=doc_utils.VisualizationDocUtil( base_path=tmp_path, base_url=doc_utils.DocUtilPaths.fig_base_url, ), df_doc_util=doc_utils.DataframeDocUtil( base_path=tmp_path, base_url=doc_utils.DocUtilPaths.df_base_url, ), nightly_doc_util=None, )
def iter_documentation_builders( datasets: Optional[List[str]] = None, *, doc_util_paths: doc_utils.DocUtilPaths = None, ) -> Iterator[BuilderDocumentation]: """Create dataset documentation string for given datasets. Args: datasets: list of datasets for which to create documentation. If None, then all available datasets will be used. doc_util_paths: Additional path for visualization, nightly info,... Yields: builder_documetation: The documentation information for each builder """ print('Retrieving the list of builders...') datasets = datasets or _all_tfds_datasets() if doc_util_paths.fig_base_path: visu_doc_util = doc_utils.VisualizationDocUtil( base_path=doc_util_paths.fig_base_path, base_url=doc_util_paths.fig_base_url, ) else: visu_doc_util = None if doc_util_paths.df_base_path: df_doc_util = doc_utils.DataframeDocUtil( base_path=doc_util_paths.df_base_path, base_url=doc_util_paths.df_base_url, ) else: df_doc_util = None if doc_util_paths.fig_base_path: nightly_doc_util = doc_utils.NightlyDocUtil( path=doc_util_paths.nightly_path, ) else: nightly_doc_util = None document_single_builder_fn = functools.partial( _document_single_builder, visu_doc_util=visu_doc_util, df_doc_util=df_doc_util, nightly_doc_util=nightly_doc_util, ) # Document all builders print(f'Document {len(datasets)} builders...') with futures.ThreadPoolExecutor( max_workers=_WORKER_COUNT_DATASETS) as tpool: tasks = [ tpool.submit(document_single_builder_fn, name) for name in datasets ] for future in tqdm.tqdm(futures.as_completed(tasks), total=len(tasks)): builder_doc = future.result() if builder_doc is None: # Builder filtered continue else: tqdm.tqdm.write( f'Documentation generated for {builder_doc.name}...') yield builder_doc print('All builder documentations generated!')