def _get_parser():
available_packages_list = " * " + "\n * ".join(get_available_packages())
parser = argparse.ArgumentParser(
description='Builds documentation and runs spell checking',
epilog=f"List of supported documentation packages:\n{available_packages_list}",
)
parser.formatter_class = argparse.RawTextHelpFormatter
parser.add_argument(
'--disable-checks', dest='disable_checks', action='store_true', help='Disables extra checks'
)
parser.add_argument(
'--one-pass-only',
dest='one_pass_only',
action='store_true',
help='Do not attempt multiple builds on error',
)
parser.add_argument(
"--package-filter",
action="append",
help=(
"Filter specifying for which packages the documentation is to be built. Wildcard are supported."
),
)
parser.add_argument('--docs-only', dest='docs_only', action='store_true', help='Only build documentation')
parser.add_argument(
'--spellcheck-only', dest='spellcheck_only', action='store_true', help='Only perform spellchecking'
)
parser.add_argument(
'--for-production',
dest='for_production',
action='store_true',
help='Builds documentation for official release i.e. all links point to stable version',
)
parser.add_argument(
"-j",
"--jobs",
dest='jobs',
type=int,
default=0,
help=(
"""\
Number of parallel processes that will be spawned to build the docs.
If passed 0, the value will be determined based on the number of CPUs.
"""
),
)
parser.add_argument(
"-v",
"--verbose",
dest='verbose',
action='store_true',
help=(
'Increases the verbosity of the script i.e. always displays a full log of '
'the build process, not just when it encounters errors'
),
)
return parser
def _get_parser():
available_packages_list = " * " + "\n * ".join(get_available_packages())
parser = argparse.ArgumentParser(
description='Builds documentation and runs spell checking',
epilog=
f"List of supported documentation packages:\n{available_packages_list}",
)
parser.formatter_class = argparse.RawTextHelpFormatter
parser.add_argument('--disable-checks',
dest='disable_checks',
action='store_true',
help='Disables extra checks')
parser.add_argument(
"--package-filter",
action="append",
help=
("Filter specifying for which packages the documentation is to be built. Wildcard are supported."
),
)
parser.add_argument('--docs-only',
dest='docs_only',
action='store_true',
help='Only build documentation')
parser.add_argument('--spellcheck-only',
dest='spellcheck_only',
action='store_true',
help='Only perform spellchecking')
parser.add_argument(
'--for-production',
dest='for_production',
action='store_true',
help=
'Builds documentation for official release i.e. all links point to stable version',
)
parser.add_argument(
"-v",
"--verbose",
dest='verbose',
action='store_true',
help=
('Increases the verbosity of the script i.e. always displays a full log of '
'the build process, not just when it encounters errors'),
)
return parser
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
if not package_filters:
_promote_new_flags()
with with_group("Available packages"):
for pkg in available_packages:
print(f" - {pkg}")
if package_filters:
print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages,
package_filters)
with with_group(
f"Documentation will be built for {len(current_packages)} package(s)"
):
for pkg_no, pkg in enumerate(current_packages, start=1):
print(f"{pkg_no}. {pkg}")
with with_group("Fetching inventories"):
fetch_inventories()
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = lint_checks.run_all_check()
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
if not package_filters:
_promote_new_flags()
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
with with_group("Available packages"):
for pkg in sorted(available_packages):
console.print(f" - {pkg}")
if package_filters:
console.print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages,
package_filters)
with with_group("Fetching inventories"):
# Inventories that could not be retrieved should be built first. This may mean this is a
# new package.
packages_without_inventories = fetch_inventories()
normal_packages, priority_packages = partition(
lambda d: d in packages_without_inventories, current_packages)
normal_packages, priority_packages = list(normal_packages), list(
priority_packages)
jobs = args.jobs if args.jobs != 0 else os.cpu_count()
with with_group(
f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs"
):
for pkg_no, pkg in enumerate(current_packages, start=1):
console.print(f"{pkg_no}. {pkg}")
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
if priority_packages:
# Build priority packages
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=priority_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
# Build normal packages
# If only one inventory is missing, the remaining packages are correct. If we are missing
# two or more inventories, it is better to try to build for all packages as the previous packages
# may have failed as well.
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages
if len(priority_packages) > 1 else normal_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
# Build documentation for some packages again if it can help them.
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = lint_checks.run_all_check()
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
if not package_filters:
_promote_new_flags()
if os.path.exists(PROVIDER_INIT_FILE):
os.remove(PROVIDER_INIT_FILE)
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
with with_group("Available packages"):
for pkg in available_packages:
print(f" - {pkg}")
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
print("Current package filters: ", package_filters)
current_packages = ([
p for p in available_packages if any(
fnmatch.fnmatch(p, f) for f in package_filters)
] if package_filters else available_packages)
with with_group(
f"Documentation will be built for {len(current_packages)} package(s)"
):
for pkg in current_packages:
print(f" - {pkg}")
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = []
general_errors.extend(
lint_checks.check_guide_links_in_operator_descriptions())
general_errors.extend(lint_checks.check_enforce_code_block())
general_errors.extend(
lint_checks.check_exampleinclude_for_example_dags())
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
print_build_errors_and_exit(
"The documentation has errors.",
all_build_errors,
all_spelling_errors,
)
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
with with_group("Available packages"):
for pkg in sorted(available_packages):
console.print(f" - {pkg}")
if package_filters:
console.print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages,
package_filters)
with with_group("Fetching inventories"):
# Inventories that could not be retrieved should be retrieved first. This may mean this is a
# new package.
priority_packages = fetch_inventories()
current_packages = sorted(current_packages,
key=lambda d: -1
if d in priority_packages else 1)
jobs = args.jobs if args.jobs != 0 else os.cpu_count()
with with_group(
f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs"
):
for pkg_no, pkg in enumerate(current_packages, start=1):
console.print(f"{pkg_no}. {pkg}")
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = lint_checks.run_all_check()
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
if not package_filters:
_promote_new_flags()
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
def _get_parser():
available_packages_list = " * " + "\n * ".join(get_available_packages())
parser = argparse.ArgumentParser(
description='Builds documentation and runs spell checking',
epilog=
f"List of supported documentation packages:\n{available_packages_list}",
)
parser.formatter_class = argparse.RawTextHelpFormatter
parser.add_argument('--disable-checks',
dest='disable_checks',
action='store_true',
help='Disables extra checks')
parser.add_argument(
"--package-filter",
action="append",
help=
("Filter specifying for which packages the documentation is to be built. Wildcard are supported."
),
)
parser.add_argument('--docs-only',
dest='docs_only',
action='store_true',
help='Only build documentation')
parser.add_argument('--spellcheck-only',
dest='spellcheck_only',
action='store_true',
help='Only perform spellchecking')
parser.add_argument(
'--for-production',
dest='for_production',
action='store_true',
help=
'Builds documentation for official release i.e. all links point to stable version',
)
parser.add_argument(
"-j",
"--jobs",
dest='jobs',
type=int,
default=1,
help=("""
Number of parallel processes that will be spawned to build the docs.
This is usually used in CI system only. Though you can also use it to run complete check
of the documntation locally if you have powerful local machine.
Default is 1 - which means that doc check runs sequentially, This is the default behaviour
because autoapi extension we use is not capable of running parallel builds at the same time using
the same source files.
In parallel builds we are using dockerised version of image built from local sources but the image
has to be prepared locally (similarly as it is in CI) before you run the docs build. Any changes you
have done locally after building the image, will not be checked.
Typically you run parallel build in this way if you want to quickly run complete check for all docs:
./breeze build-image --python 3.6
./docs/build-docs.py -j 0
"""),
)
parser.add_argument(
"-v",
"--verbose",
dest='verbose',
action='store_true',
help=
('Increases the verbosity of the script i.e. always displays a full log of '
'the build process, not just when it encounters errors'),
)
return parser
