def build_docs(name_pair,output_dir,code_url_prefix, search_hints, gen_report):
"""Build api docs for w&b.
Args:
name_pair: Name of the pymodule
output_dir: A string path, where to put the files.
code_url_prefix: prefix for "Defined in" links.
search_hints: Bool. Include meta-data search hints at the top of each file.
gen_report: Bool. Generates an API report containing the health of the
docstrings of the public API.
"""
# This is to help not document the parent class methods
for cls in [wandb.data_types.WBValue, wandb.data_types.Media, wandb.data_types.BatchableMedia, wandb.apis.public.Paginator]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=["__init__"])
doc_generator = generate_lib.DocGenerator(
root_title="W&B",
py_modules=[name_pair],
base_dir=path.dirname(wandb.__file__),
search_hints=search_hints,
code_url_prefix=code_url_prefix,
site_path="",
gen_report=gen_report,
yaml_toc=False)
doc_generator.build(output_dir)
def generate_cirq_aqt():
# This try-catch can go after v0.12 is released
try:
# should be present in the nightly (pre-release) build
import cirq_aqt
except ImportError:
# as cirq.aqt is currently not being generated anyway
# we won't handle this case (the stable build)
return
doc_generator = generate_lib.DocGenerator(
root_title="Cirq-aqt",
py_modules=[("cirq_aqt", cirq_aqt)],
base_dir=os.path.dirname(cirq_aqt.__file__),
code_url_prefix=FLAGS.code_url_prefix + "/cirq-aqt/cirq_aqt",
search_hints=FLAGS.search_hints,
site_path=FLAGS.site_path,
callbacks=[
public_api.local_definitions_filter,
filter_unwanted_inherited_methods
],
extra_docs=_doc.RECORDED_CONST_DOCS,
)
doc_controls.decorate_all_class_attributes(
doc_controls.do_not_doc_inheritable, networkx.DiGraph, skip=[])
doc_generator.build(output_dir=FLAGS.output_dir)
def main(unused_argv):
doc_generator = generate_lib.DocGenerator(
root_title="Cirq",
py_modules=[("cirq", cirq)],
base_dir=os.path.dirname(cirq.__file__),
code_url_prefix=FLAGS.code_url_prefix,
search_hints=FLAGS.search_hints,
site_path=FLAGS.site_path,
callbacks=[
public_api.local_definitions_filter,
filter_unwanted_inherited_methods
],
private_map={
# Opt to not build docs for these paths for now since they error.
"cirq.google.engine.client.quantum.QuantumEngineServiceClient":
["enums"],
"cirq.google.engine.client.quantum_v1alpha1.QuantumEngineServiceClient":
["enums"],
"cirq.google.api": ["v1"],
},
extra_docs=_doc.RECORDED_CONST_DOCS,
)
doc_controls.decorate_all_class_attributes(
doc_controls.do_not_doc_inheritable, networkx.DiGraph, skip=[])
doc_generator.build(output_dir=FLAGS.output_dir)
def generate_cirq():
doc_generator = generate_lib.DocGenerator(
root_title="Cirq",
py_modules=[("cirq", cirq)],
base_dir=os.path.dirname(cirq.__file__),
code_url_prefix=FLAGS.code_url_prefix + "/cirq-core/cirq",
search_hints=FLAGS.search_hints,
site_path=FLAGS.site_path,
callbacks=[public_api.local_definitions_filter, filter_unwanted_inherited_methods],
extra_docs=_doc.RECORDED_CONST_DOCS,
)
doc_controls.decorate_all_class_attributes(
doc_controls.do_not_doc_inheritable, networkx.DiGraph, skip=[]
)
doc_generator.build(output_dir=FLAGS.output_dir)
def main(_):
for cls in [tf.Module, tf.keras.layers.Layer]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=['__init__'])
doc_generator = generate_lib.DocGenerator(
root_title='TF-Agents',
py_modules=[('tf_agents', tf_agents)],
base_dir=os.path.dirname(tf_agents.__file__),
code_url_prefix=FLAGS.code_url_prefix,
search_hints=FLAGS.search_hints,
site_path=FLAGS.site_path,
private_map={},
callbacks=[public_api.local_definitions_filter])
sys.exit(doc_generator.build(output_dir=FLAGS.output_dir))
def generate_cirq():
doc_generator = generate_lib.DocGenerator(
root_title="Cirq",
py_modules=[("cirq", cirq)],
base_dir=os.path.dirname(cirq.__file__),
code_url_prefix=FLAGS.code_url_prefix + "/cirq-core/cirq",
search_hints=FLAGS.search_hints,
site_path=FLAGS.site_path,
callbacks=[
public_api.local_definitions_filter,
filter_unwanted_inherited_methods,
filter_type_checking,
],
extra_docs=_doc.RECORDED_CONST_DOCS,
# Submodules that we do not wish to document.
private_map={
'cirq': [
'circuits',
'devices',
'experiments',
'interop',
'linalg',
'ops',
'optimizers',
'transformers',
'qis',
'sim',
'study',
'value',
'protocols',
'ion',
'neutral_atoms',
'vis',
'work',
'json_resolver_cache',
]
},
)
doc_controls.decorate_all_class_attributes(
doc_controls.do_not_doc_inheritable, networkx.DiGraph, skip=[])
doc_generator.build(output_dir=FLAGS.output_dir)
def execute(output_dir, code_url_prefix, search_hints, site_path):
"""Builds API docs for TensorFlow Model Remediation."""
# Hide `Model` methods with a few exceptions.
for cls in [tf.Module, tf.keras.layers.Layer, tf.keras.Model]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=["__init__", "save", "compile", "call"])
# Hide `Loss` methods with a few exceptions.
for cls in [tf.keras.losses.Loss]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=["__init__", "call"])
# Hide `MinDiffLoss` and `MinDiffKernel` __call__ method.
for cls in [
tfmr.min_diff.losses.MinDiffLoss,
tfmr.min_diff.losses.MinDiffKernel
]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=["__init__"])
# Get around the decorator on Layer.call
delattr(tf.keras.layers.Layer.call,
"_tf_docs_tools_for_subclass_implementers")
# Delete common module when documenting. There is nothing there for users
# quite yet.
del tfmr.common
try:
del tfmr.tools
except AttributeError:
pass
doc_generator = generate_lib.DocGenerator(
root_title="TensorFlow Model Remediation",
py_modules=[("model_remediation", tfmr)],
base_dir=os.path.dirname(tfmr.__file__),
search_hints=search_hints,
code_url_prefix=code_url_prefix,
site_path=site_path,
callbacks=[
public_api.explicit_package_contents_filter,
public_api.local_definitions_filter
])
doc_generator.build(output_dir)
def build_docs(output_dir, code_url_prefix, search_hints):
"""Build api docs for tensorflow v2.
Args:
output_dir: A string path, where to put the files.
code_url_prefix: prefix for "Defined in" links.
search_hints: Bool. Include meta-data search hints at the top of each file.
"""
if distutils.version.LooseVersion(tf.__version__) >= "2.9":
doc_controls.set_deprecated(tf.keras.preprocessing)
# The custom page will be used for raw_ops.md not the one generated above.
doc_controls.set_custom_page_builder_cls(tf.raw_ops, RawOpsPageInfo)
# Hide raw_ops from search.
for name, obj in tf_inspect.getmembers(tf.raw_ops):
if not name.startswith("_"):
doc_controls.hide_from_search(obj)
for cls in [
tf.Module, tf.keras.layers.Layer, tf.keras.optimizers.Optimizer
]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=["__init__"])
do_not_document = [
"tf.__internal__", "tf.keras.__internal__", "tf.__operators__",
"tf.tools", "tf.compat.v1.pywrap_tensorflow", "tf.pywrap_tensorflow",
"tf.flags", "tf.batch_mat_mul_v3", "tf.sparse_segment_sum_grad"
]
for path in do_not_document:
item = tf
for part in path.split(".")[1:]:
item = getattr(item, part, None)
if item is None:
continue
doc_controls.do_not_generate_docs(item)
base_dirs, code_url_prefixes = base_dir.get_base_dirs_and_prefixes(
code_url_prefix)
doc_generator = generate_lib.DocGenerator(
root_title="TensorFlow 2",
py_modules=[("tf", tf)],
base_dir=base_dirs,
search_hints=search_hints,
code_url_prefix=code_url_prefixes,
site_path=FLAGS.site_path,
visitor_cls=TfExportAwareVisitor,
private_map=_PRIVATE_MAP,
extra_docs=_EXTRA_DOCS)
doc_generator.build(output_dir)
out_path = pathlib.Path(output_dir)
expected_path_contents = {
"tf/summary/audio.md": "tensorboard/plugins/audio/summary_v2.py",
"tf/estimator/DNNClassifier.md":
"tensorflow_estimator/python/estimator/canned/dnn.py",
"tf/nn/sigmoid_cross_entropy_with_logits.md": "python/ops/nn_impl.py",
"tf/keras/Model.md": "keras/engine/training.py",
}
all_passed = True
error_msg_parts = [
'Some "view source" links seem to be broken, please check:'
]
for (rel_path, contents) in expected_path_contents.items():
path = out_path / rel_path
if contents not in path.read_text():
all_passed = False
error_msg_parts.append(" " + str(path))
if not all_passed:
raise ValueError("\n".join(error_msg_parts))
rejected_path_contents = {
"tf/keras/optimizers.md": "keras/optimizers/__init__.py",
}
all_passed = True
error_msg_parts = [
'Bad "view source" links in generated files, please check:'
]
for rel_path, content in rejected_path_contents.items():
path = out_path / rel_path
if content in path.read_text():
all_passed = False
error_msg_parts.append(" " + str(path))
if not all_passed:
raise ValueError("\n".join(error_msg_parts))
num_files = len(list(out_path.rglob("*")))
if num_files < MIN_NUM_FILES_EXPECTED:
raise ValueError(
f"The TensorFlow api should be more than {MIN_NUM_FILES_EXPECTED} files"
f"(found {num_files}).")
def build_docs(output_dir, code_url_prefix, search_hints, gen_report):
"""Build api docs for tensorflow v2.
Args:
output_dir: A string path, where to put the files.
code_url_prefix: prefix for "Defined in" links.
search_hints: Bool. Include meta-data search hints at the top of each file.
gen_report: Bool. Generates an API report containing the health of the
docstrings of the public API.
"""
# The custom page will be used for raw_ops.md not the one generated above.
doc_controls.set_custom_page_content(tf.raw_ops, generate_raw_ops_doc())
# Hide raw_ops from search.
for name, obj in tf_inspect.getmembers(tf.raw_ops):
if not name.startswith("_"):
doc_controls.hide_from_search(obj)
for cls in [
tf.Module, tf.keras.layers.Layer, tf.keras.optimizers.Optimizer
]:
doc_controls.decorate_all_class_attributes(
decorator=doc_controls.do_not_doc_in_subclasses,
cls=cls,
skip=["__init__"])
try:
doc_controls.do_not_generate_docs(tf.__internal__)
except AttributeError:
pass
try:
doc_controls.do_not_generate_docs(tf.__operators__)
except AttributeError:
pass
try:
doc_controls.do_not_generate_docs(tf.tools)
except AttributeError:
pass
try:
doc_controls.do_not_generate_docs(tf.compat.v1.pywrap_tensorflow)
except AttributeError:
pass
try:
doc_controls.do_not_generate_docs(tf.pywrap_tensorflow)
except AttributeError:
pass
try:
doc_controls.do_not_generate_docs(tf.flags)
except AttributeError:
pass
base_dirs, code_url_prefixes = base_dir.get_base_dirs_and_prefixes(
code_url_prefix)
doc_generator = generate_lib.DocGenerator(
root_title="TensorFlow 2",
py_modules=[("tf", tf)],
base_dir=base_dirs,
search_hints=search_hints,
code_url_prefix=code_url_prefixes,
site_path=FLAGS.site_path,
visitor_cls=TfExportAwareVisitor,
private_map=_PRIVATE_MAP,
gen_report=gen_report,
)
doc_generator.build(output_dir)
if gen_report:
return
out_path = pathlib.Path(output_dir)
expected_path_contents = {
"tf/summary/audio.md": "tensorboard/plugins/audio/summary_v2.py",
"tf/estimator/DNNClassifier.md":
"tensorflow_estimator/python/estimator/canned/dnn.py",
"tf/nn/sigmoid_cross_entropy_with_logits.md": "python/ops/nn_impl.py",
"tf/keras/Model.md": "tensorflow/python/keras/engine/training.py",
}
all_passed = True
error_msg_parts = [
'Some "view source" links seem to be broken, please check:'
]
for (rel_path, contents) in expected_path_contents.items():
path = out_path / rel_path
if contents not in path.read_text():
all_passed = False
error_msg_parts.append(" " + str(path))
if not all_passed:
raise ValueError("\n".join(error_msg_parts))
rejected_path_contents = {
"tf/keras/optimizers.md": "keras/optimizers/__init__.py",
}
all_passed = True
error_msg_parts = [
'Bad "view source" links in generated files, please check:'
]
for rel_path, content in rejected_path_contents.items():
path = out_path / rel_path
if content in path.read_text():
all_passed = False
error_msg_parts.append(" " + str(path))
if not all_passed:
raise ValueError("\n".join(error_msg_parts))
num_files = len(list(out_path.rglob("*")))
if num_files < 2000:
raise ValueError("The TensorFlow api should be more than 2000 files"
"(found {}).".format(num_files))
