def main(): """Main code""" args = _get_parser().parse_args() available_packages = get_available_packages() print("Available packages: ", available_packages) docs_only = args.docs_only spellcheck_only = args.spellcheck_only disable_checks = args.disable_checks package_filter = args.package_filter print("Current package filter: ", package_filter) current_packages = (fnmatch.filter(available_packages, package_filter) if package_filter else available_packages) print( f"Documentation will be built for {len(current_packages)} package(s): {current_packages}" ) all_build_errors: Dict[Optional[str], List[DocBuildError]] = {} all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {} package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=current_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) if not disable_checks: general_errors = [] general_errors.extend( lint_checks.check_guide_links_in_operator_descriptions()) general_errors.extend(lint_checks.check_enforce_code_block()) general_errors.extend( lint_checks.check_exampleinclude_for_example_dags()) if general_errors: all_build_errors[None] = general_errors dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html") print_build_errors_and_exit( "The documentation has errors.", all_build_errors, all_spelling_errors, )
def main(): """Main code""" args = _get_parser().parse_args() available_packages = get_available_packages() docs_only = args.docs_only spellcheck_only = args.spellcheck_only disable_checks = args.disable_checks package_filters = args.package_filter for_production = args.for_production if not package_filters: _promote_new_flags() with with_group("Available packages"): for pkg in available_packages: print(f" - {pkg}") if package_filters: print("Current package filters: ", package_filters) current_packages = process_package_filters(available_packages, package_filters) with with_group( f"Documentation will be built for {len(current_packages)} package(s)" ): for pkg_no, pkg in enumerate(current_packages, start=1): print(f"{pkg_no}. {pkg}") with with_group("Fetching inventories"): fetch_inventories() all_build_errors: Dict[Optional[str], List[DocBuildError]] = {} all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {} package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=current_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, for_production=for_production, verbose=args.verbose, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) to_retry_packages = [ package_name for package_name, errors in package_build_errors.items() if any( any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD) for e in errors) ] if to_retry_packages: for package_name in to_retry_packages: if package_name in all_build_errors: del all_build_errors[package_name] if package_name in all_spelling_errors: del all_spelling_errors[package_name] package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=to_retry_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, for_production=for_production, verbose=args.verbose, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) if not disable_checks: general_errors = lint_checks.run_all_check() if general_errors: all_build_errors[None] = general_errors dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html") if not package_filters: _promote_new_flags() print_build_errors_and_exit( all_build_errors, all_spelling_errors, )
def main(): """Main code""" args = _get_parser().parse_args() available_packages = get_available_packages() docs_only = args.docs_only spellcheck_only = args.spellcheck_only disable_checks = args.disable_checks package_filters = args.package_filter for_production = args.for_production with with_group("Available packages"): for pkg in sorted(available_packages): console.print(f" - {pkg}") if package_filters: console.print("Current package filters: ", package_filters) current_packages = process_package_filters(available_packages, package_filters) with with_group("Fetching inventories"): # Inventories that could not be retrieved should be built first. This may mean this is a # new package. packages_without_inventories = fetch_inventories() normal_packages, priority_packages = partition( lambda d: d in packages_without_inventories, current_packages) normal_packages, priority_packages = list(normal_packages), list( priority_packages) jobs = args.jobs if args.jobs != 0 else os.cpu_count() with with_group( f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs" ): for pkg_no, pkg in enumerate(current_packages, start=1): console.print(f"{pkg_no}. {pkg}") all_build_errors: Dict[Optional[str], List[DocBuildError]] = {} all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {} if priority_packages: # Build priority packages package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=priority_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, for_production=for_production, jobs=jobs, verbose=args.verbose, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) # Build normal packages # If only one inventory is missing, the remaining packages are correct. If we are missing # two or more inventories, it is better to try to build for all packages as the previous packages # may have failed as well. package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=current_packages if len(priority_packages) > 1 else normal_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, for_production=for_production, jobs=jobs, verbose=args.verbose, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) # Build documentation for some packages again if it can help them. to_retry_packages = [ package_name for package_name, errors in package_build_errors.items() if any( any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD) for e in errors) ] if to_retry_packages: for package_name in to_retry_packages: if package_name in all_build_errors: del all_build_errors[package_name] if package_name in all_spelling_errors: del all_spelling_errors[package_name] package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=to_retry_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, for_production=for_production, jobs=jobs, verbose=args.verbose, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) if not disable_checks: general_errors = lint_checks.run_all_check() if general_errors: all_build_errors[None] = general_errors dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html") if not package_filters: _promote_new_flags() if os.path.exists(PROVIDER_INIT_FILE): os.remove(PROVIDER_INIT_FILE) print_build_errors_and_exit( all_build_errors, all_spelling_errors, )
def main(): """Main code""" args = _get_parser().parse_args() available_packages = get_available_packages() print("Available packages: ", available_packages) docs_only = args.docs_only spellcheck_only = args.spellcheck_only disable_checks = args.disable_checks package_filters = args.package_filter print("Current package filters: ", package_filters) current_packages = ( [p for p in available_packages if any(fnmatch.fnmatch(p, f) for f in package_filters)] if package_filters else available_packages ) print(f"Documentation will be built for {len(current_packages)} package(s): {current_packages}") all_build_errors: Dict[Optional[str], List[DocBuildError]] = {} all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {} package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=current_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) to_retry_packages = [ package_name for package_name, errors in package_build_errors.items() if any(any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD) for e in errors) ] if to_retry_packages: for package_name in to_retry_packages: if package_name in all_build_errors: del all_build_errors[package_name] if package_name in all_spelling_errors: del all_spelling_errors[package_name] package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=to_retry_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) if not disable_checks: general_errors = [] general_errors.extend(lint_checks.check_guide_links_in_operator_descriptions()) general_errors.extend(lint_checks.check_enforce_code_block()) general_errors.extend(lint_checks.check_exampleinclude_for_example_dags()) if general_errors: all_build_errors[None] = general_errors dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html") print_build_errors_and_exit( "The documentation has errors.", all_build_errors, all_spelling_errors, )
def main(): """Main code""" args = _get_parser().parse_args() available_packages = get_available_packages() docs_only = args.docs_only spellcheck_only = args.spellcheck_only disable_checks = args.disable_checks package_filters = args.package_filter for_production = args.for_production with with_group("Available packages"): for pkg in sorted(available_packages): console.print(f" - {pkg}") if package_filters: console.print("Current package filters: ", package_filters) current_packages = process_package_filters(available_packages, package_filters) with with_group("Fetching inventories"): # Inventories that could not be retrieved should be retrieved first. This may mean this is a # new package. priority_packages = fetch_inventories() current_packages = sorted(current_packages, key=lambda d: -1 if d in priority_packages else 1) jobs = args.jobs if args.jobs != 0 else os.cpu_count() with with_group( f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs" ): for pkg_no, pkg in enumerate(current_packages, start=1): console.print(f"{pkg_no}. {pkg}") all_build_errors: Dict[Optional[str], List[DocBuildError]] = {} all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {} package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=current_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, for_production=for_production, jobs=jobs, verbose=args.verbose, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) to_retry_packages = [ package_name for package_name, errors in package_build_errors.items() if any( any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD) for e in errors) ] if to_retry_packages: for package_name in to_retry_packages: if package_name in all_build_errors: del all_build_errors[package_name] if package_name in all_spelling_errors: del all_spelling_errors[package_name] package_build_errors, package_spelling_errors = build_docs_for_packages( current_packages=to_retry_packages, docs_only=docs_only, spellcheck_only=spellcheck_only, for_production=for_production, jobs=jobs, verbose=args.verbose, ) if package_build_errors: all_build_errors.update(package_build_errors) if package_spelling_errors: all_spelling_errors.update(package_spelling_errors) if not disable_checks: general_errors = lint_checks.run_all_check() if general_errors: all_build_errors[None] = general_errors dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html") if not package_filters: _promote_new_flags() print_build_errors_and_exit( all_build_errors, all_spelling_errors, )