def document_single_builder_fn(tmp_path):
yield functools.partial(
document_datasets._document_single_builder,
visu_doc_util=doc_utils.VisualizationDocUtil(
base_path=tmp_path,
base_url=doc_utils.DocUtilPaths.fig_base_url,
),
df_doc_util=doc_utils.DataframeDocUtil(
base_path=tmp_path,
base_url=doc_utils.DocUtilPaths.df_base_url,
),
nightly_doc_util=None,
)
def iter_documentation_builders(
datasets: Optional[List[str]] = None,
*,
doc_util_paths: doc_utils.DocUtilPaths = None,
) -> Iterator[BuilderDocumentation]:
"""Create dataset documentation string for given datasets.
Args:
datasets: list of datasets for which to create documentation.
If None, then all available datasets will be used.
doc_util_paths: Additional path for visualization, nightly info,...
Yields:
builder_documetation: The documentation information for each builder
"""
print('Retrieving the list of builders...')
datasets = datasets or _all_tfds_datasets()
if doc_util_paths.fig_base_path:
visu_doc_util = doc_utils.VisualizationDocUtil(
base_path=doc_util_paths.fig_base_path,
base_url=doc_util_paths.fig_base_url,
)
else:
visu_doc_util = None
if doc_util_paths.df_base_path:
df_doc_util = doc_utils.DataframeDocUtil(
base_path=doc_util_paths.df_base_path,
base_url=doc_util_paths.df_base_url,
)
else:
df_doc_util = None
if doc_util_paths.fig_base_path:
nightly_doc_util = doc_utils.NightlyDocUtil(
path=doc_util_paths.nightly_path, )
else:
nightly_doc_util = None
document_single_builder_fn = functools.partial(
_document_single_builder,
visu_doc_util=visu_doc_util,
df_doc_util=df_doc_util,
nightly_doc_util=nightly_doc_util,
)
# Document all builders
print(f'Document {len(datasets)} builders...')
with futures.ThreadPoolExecutor(
max_workers=_WORKER_COUNT_DATASETS) as tpool:
tasks = [
tpool.submit(document_single_builder_fn, name) for name in datasets
]
for future in tqdm.tqdm(futures.as_completed(tasks), total=len(tasks)):
builder_doc = future.result()
if builder_doc is None: # Builder filtered
continue
else:
tqdm.tqdm.write(
f'Documentation generated for {builder_doc.name}...')
yield builder_doc
print('All builder documentations generated!')