def _format_action(self, action: Action):
if isinstance(action, argparse._SubParsersAction): # pylint: disable=protected-access
parts = []
subactions = action._get_subactions() # pylint: disable=protected-access
action_subcommnads, group_subcommnands = partition(
lambda d: isinstance(ALL_COMMANDS_DICT[d.dest], GroupCommand), subactions
)
parts.append("\n")
parts.append('%*s%s:\n' % (self._current_indent, '', "Groups"))
self._indent()
for subaction in group_subcommnands:
parts.append(self._format_action(subaction))
self._dedent()
parts.append("\n")
parts.append('%*s%s:\n' % (self._current_indent, '', "Commands"))
self._indent()
for subaction in action_subcommnads:
parts.append(self._format_action(subaction))
self._dedent()
# return a single string
return self._join_parts(parts)
return super()._format_action(action)
def display_recursive(
prefix: List[str],
commands: Iterable[Union[GroupCommand, ActionCommand]],
help_msg: Optional[str] = None,
):
actions: List[ActionCommand]
groups: List[GroupCommand]
actions_iter, groups_iter = partition(
lambda x: isinstance(x, GroupCommand), commands)
actions, groups = list(actions_iter), list(groups_iter)
console = Console()
if actions:
table = SimpleTable(title=help_msg or "Miscellaneous commands")
table.add_column(width=40)
table.add_column()
for action_command in sorted(actions, key=lambda d: d.name):
table.add_row(" ".join([*prefix, action_command.name]),
action_command.help)
console.print(table)
if groups:
for group_command in sorted(groups, key=lambda d: d.name):
group_prefix = [*prefix, group_command.name]
display_recursive(group_prefix, group_command.subcommands,
group_command.help)
def display_recursive(prefix: List[str],
commands: Iterable[Union[GroupCommand,
ActionCommand]]):
actions: List[ActionCommand]
groups: List[GroupCommand]
actions_tter, groups_iter = partition(
lambda x: isinstance(x, GroupCommand), commands)
actions, groups = list(actions_tter), list(groups_iter)
if actions:
for action_command in sorted(actions, key=lambda d: d.name):
print(" ", end="")
cprint(" ".join([*prefix, action_command.name]),
attrs=["bold"],
end="")
print(f" - {action_command.help}")
print()
if groups:
for group_command in sorted(groups, key=lambda d: d.name):
group_prefix = [*prefix, group_command.name]
# print(bold(" ".join(group_prefix)), end="")
cprint(group_command.help, attrs=["bold", "underline"])
print()
display_recursive(group_prefix, group_command.subcommands)
print()
def fetch_inventories():
"""Fetch all inventories for Airflow documentation packages and store in cache."""
os.makedirs(os.path.dirname(CACHE_DIR), exist_ok=True)
to_download: List[Tuple[str, str, str]] = []
for pkg_name in get_available_providers_packages():
to_download.append((
pkg_name,
S3_DOC_URL_VERSIONED.format(package_name=pkg_name),
f'{CACHE_DIR}/{pkg_name}/objects.inv',
))
for pkg_name in ['apache-airflow', 'helm-chart']:
to_download.append((
pkg_name,
S3_DOC_URL_VERSIONED.format(package_name=pkg_name),
f'{CACHE_DIR}/{pkg_name}/objects.inv',
))
for pkg_name in ['apache-airflow-providers', 'docker-stack']:
to_download.append((
pkg_name,
S3_DOC_URL_NON_VERSIONED.format(package_name=pkg_name),
f'{CACHE_DIR}/{pkg_name}/objects.inv',
))
to_download.extend((
pkg_name,
f"{doc_url}/objects.inv",
f'{CACHE_DIR}/{pkg_name}/objects.inv',
) for pkg_name, doc_url in THIRD_PARTY_INDEXES.items())
to_download = [(pkg_name, url, path) for pkg_name, url, path in to_download
if _is_outdated(path)]
if not to_download:
print("Nothing to do")
return []
print(f"To download {len(to_download)} inventorie(s)")
with requests.Session() as session, concurrent.futures.ThreadPoolExecutor(
DEFAULT_POOLSIZE) as pool:
download_results: Iterator[Tuple[str, bool]] = pool.map(
_fetch_file,
repeat(session, len(to_download)),
(pkg_name for pkg_name, _, _ in to_download),
(url for _, url, _ in to_download),
(path for _, _, path in to_download),
)
failed, success = partition(lambda d: d[1], download_results)
failed, success = list(failed), list(success)
print(f"Result: {len(success)} success, {len(failed)} failed")
if failed:
print("Failed packages:")
for pkg_no, (pkg_name, _) in enumerate(failed, start=1):
print(f"{pkg_no}. {pkg_name}")
return [pkg_name for pkg_name, status in failed]
def _sort_args(args: Iterable[Arg]) -> Iterable[Arg]:
"""
Sort subcommand optional args, keep positional args
"""
def get_long_option(arg: Arg):
"""
Get long option from Arg.flags
"""
return arg.flags[0] if len(arg.flags) == 1 else arg.flags[1]
positional, optional = partition(lambda x: x.flags[0].startswith("-"), args)
yield from positional
yield from sorted(optional, key=lambda x: get_long_option(x).lower())
def main():
"""Main code"""
args = _get_parser().parse_args()
available_packages = get_available_packages()
docs_only = args.docs_only
spellcheck_only = args.spellcheck_only
disable_checks = args.disable_checks
package_filters = args.package_filter
for_production = args.for_production
with with_group("Available packages"):
for pkg in sorted(available_packages):
console.print(f" - {pkg}")
if package_filters:
console.print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages,
package_filters)
with with_group("Fetching inventories"):
# Inventories that could not be retrieved should be built first. This may mean this is a
# new package.
packages_without_inventories = fetch_inventories()
normal_packages, priority_packages = partition(
lambda d: d in packages_without_inventories, current_packages)
normal_packages, priority_packages = list(normal_packages), list(
priority_packages)
jobs = args.jobs if args.jobs != 0 else os.cpu_count()
with with_group(
f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs"
):
for pkg_no, pkg in enumerate(current_packages, start=1):
console.print(f"{pkg_no}. {pkg}")
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
if priority_packages:
# Build priority packages
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=priority_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
# Build normal packages
# If only one inventory is missing, the remaining packages are correct. If we are missing
# two or more inventories, it is better to try to build for all packages as the previous packages
# may have failed as well.
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=current_packages
if len(priority_packages) > 1 else normal_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
# Build documentation for some packages again if it can help them.
to_retry_packages = [
package_name for package_name, errors in package_build_errors.items()
if any(
any((m in e.message) for m in ERRORS_ELIGIBLE_TO_REBUILD)
for e in errors)
]
if to_retry_packages:
for package_name in to_retry_packages:
if package_name in all_build_errors:
del all_build_errors[package_name]
if package_name in all_spelling_errors:
del all_spelling_errors[package_name]
package_build_errors, package_spelling_errors = build_docs_for_packages(
current_packages=to_retry_packages,
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
all_build_errors.update(package_build_errors)
if package_spelling_errors:
all_spelling_errors.update(package_spelling_errors)
if not disable_checks:
general_errors = lint_checks.run_all_check()
if general_errors:
all_build_errors[None] = general_errors
dev_index_generator.generate_index(f"{DOCS_DIR}/_build/index.html")
if not package_filters:
_promote_new_flags()
if os.path.exists(PROVIDER_INIT_FILE):
os.remove(PROVIDER_INIT_FILE)
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
