def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
package_filters = args.package_filter
current_packages = process_package_filters(available_packages, package_filters)
print(f"Publishing docs for {len(current_packages)} package(s)")
for pkg in current_packages:
print(f" - {pkg}")
print()
for package_name in current_packages:
builder = AirflowDocsBuilder(package_name=package_name, for_production=True)
builder.publish(override_versioned=args.override_versioned)
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
if not package_filters:
_promote_new_flags()
with with_group("Available packages"):
for pkg in available_packages:
print(f" - {pkg}")
if package_filters:
print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages,
package_filters)
with with_group(
f"Documentation will be built for {len(current_packages)} package(s)"
):
for pkg_no, pkg in enumerate(current_packages, start=1):
print(f"{pkg_no}. {pkg}")
with with_group("Fetching inventories"):
fetch_inventories()
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = lint_checks.run_all_check()
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
if not package_filters:
_promote_new_flags()
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
with with_group("Available packages"):
for pkg in sorted(available_packages):
console.print(f" - {pkg}")
if package_filters:
console.print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages,
package_filters)
with with_group("Fetching inventories"):
# Inventories that could not be retrieved should be built first. This may mean this is a
# new package.
packages_without_inventories = fetch_inventories()
normal_packages, priority_packages = partition(
lambda d: d in packages_without_inventories, current_packages)
normal_packages, priority_packages = list(normal_packages), list(
priority_packages)
jobs = args.jobs if args.jobs != 0 else os.cpu_count()
with with_group(
f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs"
):
for pkg_no, pkg in enumerate(current_packages, start=1):
console.print(f"{pkg_no}. {pkg}")
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
if priority_packages:
# Build priority packages
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=priority_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
# Build normal packages
# If only one inventory is missing, the remaining packages are correct. If we are missing
# two or more inventories, it is better to try to build for all packages as the previous packages
# may have failed as well.
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages
if len(priority_packages) > 1 else normal_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
# Build documentation for some packages again if it can help them.
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = lint_checks.run_all_check()
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
if not package_filters:
_promote_new_flags()
if os.path.exists(PROVIDER_INIT_FILE):
os.remove(PROVIDER_INIT_FILE)
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
with with_group("Available packages"):
for pkg in sorted(available_packages):
console.print(f" - {pkg}")
if package_filters:
console.print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages,
package_filters)
with with_group("Fetching inventories"):
# Inventories that could not be retrieved should be retrieved first. This may mean this is a
# new package.
priority_packages = fetch_inventories()
current_packages = sorted(current_packages,
key=lambda d: -1
if d in priority_packages else 1)
jobs = args.jobs if args.jobs != 0 else os.cpu_count()
with with_group(
f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs"
):
for pkg_no, pkg in enumerate(current_packages, start=1):
console.print(f"{pkg_no}. {pkg}")
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = lint_checks.run_all_check()
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
if not package_filters:
_promote_new_flags()
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
