def document_single_builder(builder):
"""Doc string for a single builder, with or without configs."""
print('Document builder %s...' % builder.name)
get_config_builder = lambda config: tfds.builder(builder.name,
config=config)
config_builders = []
if builder.builder_configs:
with futures.ThreadPoolExecutor(
max_workers=WORKER_COUNT_CONFIGS) as tpool:
config_builders = list(
tpool.map(get_config_builder, builder.BUILDER_CONFIGS))
visu_doc_util = VisualizationDocUtil()
df_doc_util = DataframeDocUtil()
out_str = dataset_markdown_builder.get_markdown_string(
builder=builder,
config_builders=config_builders,
visu_doc_util=visu_doc_util,
df_doc_util=df_doc_util,
nightly_doc_util=NightlyDocUtil(),
)
schema_org_tmpl = get_mako_template('schema_org')
schema_org_out_str = schema_org_tmpl.render_unicode(
builder=builder,
config_builders=config_builders,
visu_doc_util=visu_doc_util,
).strip()
out_str = schema_org_out_str + '\n' + out_str
return out_str
def _document_single_builder_inner(
name: str,
nightly_doc_util: doc_utils.NightlyDocUtil,
**kwargs: Any,
) -> Optional[BuilderDocumentation]:
"""Doc string for a single builder, with or without configs."""
tqdm.tqdm.write(f'Document builder {name}...')
doc_info = _load_builder(name)
if doc_info is None:
return None
out_str = dataset_markdown_builder.get_markdown_string(
namespace=doc_info.namespace,
builder=doc_info.builder,
config_builders=doc_info.config_builders,
nightly_doc_util=nightly_doc_util,
**kwargs)
is_nightly = bool(
nightly_doc_util and nightly_doc_util.is_builder_nightly(name))
return BuilderDocumentation(
name=name,
filestem=name.replace(':', '_'),
content=out_str,
section=doc_info.section,
is_manual=bool(doc_info.builder.MANUAL_DOWNLOAD_INSTRUCTIONS),
is_nightly=is_nightly,
)
def _document_single_builder_inner(
name: str,
visu_doc_util: doc_utils.VisualizationDocUtil,
df_doc_util: doc_utils.DataframeDocUtil,
nightly_doc_util: doc_utils.NightlyDocUtil,
) -> Optional[BuilderDocumentation]:
"""Doc string for a single builder, with or without configs."""
builder_cls = tfds.builder_cls(name)
section = _get_section(builder_cls)
tqdm.tqdm.write(f'Document builder {name}...')
builder, config_builders = _load_builder(builder_cls)
out_str = dataset_markdown_builder.get_markdown_string(
builder=builder,
config_builders=config_builders,
visu_doc_util=visu_doc_util,
df_doc_util=df_doc_util,
nightly_doc_util=nightly_doc_util,
)
is_nightly = bool(
nightly_doc_util and nightly_doc_util.is_builder_nightly(name)
)
return BuilderDocumentation(
name=name,
content=out_str,
section=section,
is_manual=bool(builder_cls.MANUAL_DOWNLOAD_INSTRUCTIONS),
is_nightly=is_nightly,
)