def load_command_table(self, args): # When executing "azdev linter --include-whl-extensions next" in CI, args is none, so judgment is added if args: from azure.cli.core.util import roughly_parse_command command = roughly_parse_command(args) if command == 'next' and not self._has_reload_command_table: self._has_reload_command_table = True from unittest.mock import patch with patch.dict("os.environ", {'AZURE_CORE_USE_COMMAND_INDEX': 'False'}): self.cli_ctx.invocation.commands_loader.load_command_table( args) from azext_next.utils import log_command_history log_command_history(command, args) from azext_next.commands import load_command_table load_command_table(self, args) return self.command_table
def load_command_table(self, args): from importlib import import_module import pkgutil import traceback from azure.cli.core.commands import (_load_module_command_loader, _load_extension_command_loader, BLOCKED_MODS, ExtensionCommandSource) from azure.cli.core.extension import (get_extensions, get_extension_path, get_extension_modname) def _update_command_table_from_modules(args, command_modules=None): """Loads command tables from modules and merge into the main command table. :param args: Arguments of the command. :param list command_modules: Command modules to load, in the format like ['resource', 'profile']. If None, will do module discovery and load all modules. If [], only ALWAYS_LOADED_MODULES will be loaded. Otherwise, the list will be extended using ALWAYS_LOADED_MODULES. """ # As command modules are built-in, the existence of modules in ALWAYS_LOADED_MODULES is NOT checked if command_modules is not None: command_modules.extend(ALWAYS_LOADED_MODULES) else: # Perform module discovery command_modules = [] try: mods_ns_pkg = import_module('azure.cli.command_modules') command_modules = [ modname for _, modname, _ in pkgutil.iter_modules( mods_ns_pkg.__path__) ] logger.debug('Discovered command modules: %s', command_modules) except ImportError as e: logger.warning(e) count = 0 cumulative_elapsed_time = 0 cumulative_group_count = 0 cumulative_command_count = 0 logger.debug("Loading command modules:") logger.debug(self.header_mod) for mod in [m for m in command_modules if m not in BLOCKED_MODS]: try: start_time = timeit.default_timer() module_command_table, module_group_table = _load_module_command_loader( self, args, mod) for cmd in module_command_table.values(): cmd.command_source = mod self.command_table.update(module_command_table) self.command_group_table.update(module_group_table) elapsed_time = timeit.default_timer() - start_time logger.debug(self.item_format_string, mod, elapsed_time, len(module_group_table), len(module_command_table)) count += 1 cumulative_elapsed_time += elapsed_time cumulative_group_count += len(module_group_table) cumulative_command_count += len(module_command_table) except Exception as ex: # pylint: disable=broad-except # Changing this error message requires updating CI script that checks for failed # module loading. import azure.cli.core.telemetry as telemetry logger.error("Error loading command module '%s': %s", mod, ex) telemetry.set_exception( exception=ex, fault_type='module-load-error-' + mod, summary='Error loading module: {}'.format(mod)) logger.debug(traceback.format_exc()) # Summary line logger.debug(self.item_format_string, "Total ({})".format(count), cumulative_elapsed_time, cumulative_group_count, cumulative_command_count) def _update_command_table_from_extensions(ext_suppressions, extension_modname=None): """Loads command tables from extensions and merge into the main command table. :param ext_suppressions: Extension suppression information. :param extension_modname: Command modules to load, in the format like ['azext_timeseriesinsights']. If None, will do extension discovery and load all extensions. If [], only ALWAYS_LOADED_EXTENSIONS will be loaded. Otherwise, the list will be extended using ALWAYS_LOADED_EXTENSIONS. If the extensions in the list are not installed, it will be skipped. """ def _handle_extension_suppressions(extensions): filtered_extensions = [] for ext in extensions: should_include = True for suppression in ext_suppressions: if should_include and suppression.handle_suppress(ext): should_include = False if should_include: filtered_extensions.append(ext) return filtered_extensions def _filter_modname(extensions): # Extension's name may not be the same as its modname. eg. name: virtual-wan, modname: azext_vwan filtered_extensions = [] for ext in extensions: ext_mod = get_extension_modname(ext.name, ext.path) # Filter the extensions according to the index if ext_mod in extension_modname: filtered_extensions.append(ext) extension_modname.remove(ext_mod) if extension_modname: logger.debug( "These extensions are not installed and will be skipped: %s", extension_modname) return filtered_extensions extensions = get_extensions() if extensions: if extension_modname is not None: extension_modname.extend(ALWAYS_LOADED_EXTENSIONS) extensions = _filter_modname(extensions) allowed_extensions = _handle_extension_suppressions(extensions) module_commands = set(self.command_table.keys()) count = 0 cumulative_elapsed_time = 0 cumulative_group_count = 0 cumulative_command_count = 0 logger.debug("Loading extensions:") logger.debug(self.header_ext) for ext in allowed_extensions: try: # Import in the `for` loop because `allowed_extensions` can be []. In such case we # don't need to import `check_version_compatibility` at all. from azure.cli.core.extension.operations import check_version_compatibility check_version_compatibility(ext.get_metadata()) except CLIError as ex: # issue warning and skip loading extensions that aren't compatible with the CLI core logger.warning(ex) continue ext_name = ext.name ext_dir = ext.path or get_extension_path(ext_name) sys.path.append(ext_dir) try: ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir) # Add to the map. This needs to happen before we load commands as registering a command # from an extension requires this map to be up-to-date. # self._mod_to_ext_map[ext_mod] = ext_name start_time = timeit.default_timer() extension_command_table, extension_group_table = \ _load_extension_command_loader(self, args, ext_mod) for cmd_name, cmd in extension_command_table.items(): cmd.command_source = ExtensionCommandSource( extension_name=ext_name, overrides_command=cmd_name in module_commands, preview=ext.preview, experimental=ext.experimental) self.command_table.update(extension_command_table) self.command_group_table.update(extension_group_table) elapsed_time = timeit.default_timer() - start_time logger.debug(self.item_ext_format_string, ext_name, elapsed_time, len(extension_group_table), len(extension_command_table), ext_dir) count += 1 cumulative_elapsed_time += elapsed_time cumulative_group_count += len(extension_group_table) cumulative_command_count += len( extension_command_table) except Exception as ex: # pylint: disable=broad-except self.cli_ctx.raise_event(EVENT_FAILED_EXTENSION_LOAD, extension_name=ext_name) logger.warning( "Unable to load extension '%s: %s'. Use --debug for more information.", ext_name, ex) logger.debug(traceback.format_exc()) # Summary line logger.debug(self.item_ext_format_string, "Total ({})".format(count), cumulative_elapsed_time, cumulative_group_count, cumulative_command_count, "") def _wrap_suppress_extension_func(func, ext): """ Wrapper method to handle centralization of log messages for extension filters """ res = func(ext) should_suppress = res reason = "Use --debug for more information." if isinstance(res, tuple): should_suppress, reason = res suppress_types = (bool, type(None)) if not isinstance(should_suppress, suppress_types): raise ValueError( "Command module authoring error: " "Valid extension suppression values are {} in {}".format( suppress_types, func)) if should_suppress: logger.warning("Extension %s (%s) has been suppressed. %s", ext.name, ext.version, reason) logger.debug( "Extension %s (%s) suppressed from being loaded due " "to %s", ext.name, ext.version, func) return should_suppress def _get_extension_suppressions(mod_loaders): res = [] for m in mod_loaders: suppressions = getattr(m, 'suppress_extension', None) if suppressions: suppressions = suppressions if isinstance( suppressions, list) else [suppressions] for sup in suppressions: if isinstance(sup, ModExtensionSuppress): res.append(sup) return res # Clear the tables to make this method idempotent self.command_group_table.clear() self.command_table.clear() command_index = None # Set fallback=False to turn off command index in case of regression use_command_index = self.cli_ctx.config.getboolean('core', 'use_command_index', fallback=True) if use_command_index: command_index = CommandIndex(self.cli_ctx) index_result = command_index.get(args) if index_result: index_modules, index_extensions = index_result # Always load modules and extensions, because some of them (like those in # ALWAYS_LOADED_EXTENSIONS) don't expose a command, but hooks into handlers in CLI core _update_command_table_from_modules(args, index_modules) # The index won't contain suppressed extensions _update_command_table_from_extensions([], index_extensions) logger.debug("Loaded %d groups, %d commands.", len(self.command_group_table), len(self.command_table)) from azure.cli.core.util import roughly_parse_command # The index may be outdated. Make sure the command appears in the loaded command table raw_cmd = roughly_parse_command(args) for cmd in self.command_table: if raw_cmd.startswith(cmd): # For commands with positional arguments, the raw command won't match the one in the # command table. For example, `az find vm create` won't exist in the command table, but the # corresponding command should be `az find`. # raw command : az find vm create # command table: az find # remaining : vm create logger.debug("Found a match in the command table.") logger.debug("Raw command : %s", raw_cmd) logger.debug("Command table: %s", cmd) remaining = raw_cmd[len(cmd) + 1:] if remaining: logger.debug("remaining : %s %s", ' ' * len(cmd), remaining) return self.command_table # For command group, it must be an exact match, as no positional argument is supported by # command group operations. if raw_cmd in self.command_group_table: logger.debug( "Found a match in the command group table for '%s'.", raw_cmd) return self.command_table logger.debug( "Could not find a match in the command or command group table for '%s'. " "The index may be outdated.", raw_cmd) else: logger.debug("No module found from index for '%s'", args) # No module found from the index. Load all command modules and extensions logger.debug("Loading all modules and extensions") _update_command_table_from_modules(args) ext_suppressions = _get_extension_suppressions(self.loaders) # We always load extensions even if the appropriate module has been loaded # as an extension could override the commands already loaded. _update_command_table_from_extensions(ext_suppressions) logger.debug("Loaded %d groups, %d commands.", len(self.command_group_table), len(self.command_table)) if use_command_index: command_index.update(self.command_table) return self.command_table
def _check_value(self, action, value): # pylint: disable=too-many-statements, too-many-locals # Override to customize the error message when a argument is not among the available choices # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: # pylint: disable=too-many-nested-blocks # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. # use cli_ctx from cli_help which is not lost. cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) caused_by_extension_not_installed = False command_name_inferred = self.prog error_msg = None if not self.command_source: candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) if candidates: # use the most likely candidate to replace the misspelled command args = self.prog.split() + self._raw_arguments args_inferred = [item if item != value else candidates[0] for item in args] command_name_inferred = ' '.join(args_inferred).split('-')[0] use_dynamic_install = self._get_extension_use_dynamic_install_config() if use_dynamic_install != 'no' and not candidates: # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command cmd_list = self.prog.split() + self._raw_arguments command_str = roughly_parse_command(cmd_list[1:]) ext_name = self._search_in_extension_commands(command_str) if ext_name: caused_by_extension_not_installed = True telemetry.set_command_details(command_str, parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access extension_name=ext_name) run_after_extension_installed = self._get_extension_run_after_dynamic_install_config() if use_dynamic_install == 'yes_without_prompt': logger.warning('The command requires the extension %s. ' 'It will be installed first.', ext_name) go_on = True else: from knack.prompting import prompt_y_n, NoTTYException prompt_msg = 'The command requires the extension {}. ' \ 'Do you want to install it now?'.format(ext_name) if run_after_extension_installed: prompt_msg = '{} The command will continue to run after the extension is installed.' \ .format(prompt_msg) NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \ "yes_without_prompt' to allow installing extensions without prompt." try: go_on = prompt_y_n(prompt_msg, default='y') if go_on: logger.warning(NO_PROMPT_CONFIG_MSG) except NoTTYException: logger.warning("The command requires the extension %s.\n " "Unable to prompt for extension install confirmation as no tty " "available. %s", ext_name, NO_PROMPT_CONFIG_MSG) go_on = False if go_on: from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name, upgrade=True) if run_after_extension_installed: import subprocess import platform exit_code = subprocess.call(cmd_list, shell=platform.system() == 'Windows') error_msg = ("Extension {} dynamically installed and commands will be " "rerun automatically.").format(ext_name) telemetry.set_user_fault(error_msg) self.exit(exit_code) else: with CommandLoggerContext(logger): error_msg = 'Extension {} installed. Please rerun your command.'.format(ext_name) logger.error(error_msg) telemetry.set_user_fault(error_msg) self.exit(2) else: error_msg = "The command requires the latest version of extension {ext_name}. " \ "To install, run 'az extension add --upgrade -n {ext_name}'.".format(ext_name=ext_name) if not error_msg: # parser has no `command_source`, value is part of command itself error_msg = "'{value}' is misspelled or not recognized by the system.".format(value=value) az_error = CommandNotFoundError(error_msg) else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[0] if action.option_strings else action.dest error_msg = "{prog}: '{value}' is not a valid value for '{param}'.".format( prog=self.prog, value=value, param=parameter) candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) az_error = InvalidArgumentValueError(error_msg) command_arguments = self._get_failure_recovery_arguments(action) if candidates: az_error.set_recommendation("Did you mean '{}' ?".format(candidates[0])) # recommend a command for user recommender = CommandRecommender(*command_arguments, error_msg, cli_ctx) recommender.set_help_examples(self.get_examples(command_name_inferred)) recommended_command = recommender.recommend_a_command() if recommended_command: az_error.set_recommendation("Try this: '{}'".format(recommended_command)) # remind user to check extensions if we can not find a command to recommend if isinstance(az_error, CommandNotFoundError) \ and not az_error.recommendations and self.prog == 'az' \ and use_dynamic_install == 'no': az_error.set_recommendation(EXTENSION_REFERENCE) az_error.set_recommendation(OVERVIEW_REFERENCE.format(command=self.prog)) if not caused_by_extension_not_installed: az_error.print_error() az_error.send_telemetry() self.exit(2)
def _check_value_in_extensions(cli_ctx, parser, args, no_prompt): # pylint: disable=too-many-statements, too-many-locals """Check if the command args can be found in extension commands. Exit command if the error is caused by an extension not installed. Otherwise return. """ # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command from azure.cli.core.azclierror import NoTTYError exit_code = 2 command_str = roughly_parse_command(args[1:]) allow_prefix_match = args[-1] == '-h' or args[-1] == '--help' ext_name = _search_in_extension_commands(cli_ctx, command_str, allow_prefix_match=allow_prefix_match) # ext_name is a list if the input command matches the prefix of one or more extension commands, # for instance: `az blueprint` when running `az blueprint -h` # ext_name is a str if the input command matches a complete command of an extension, # for instance: `az blueprint create` if isinstance(ext_name, list): if len(ext_name) > 1: from knack.prompting import prompt_choice_list, NoTTYException prompt_msg = "The command requires the latest version of one of the following " \ "extensions. You need to pick one to install:" try: choice_idx = prompt_choice_list(prompt_msg, ext_name) ext_name = ext_name[choice_idx] no_prompt = True except NoTTYException: tty_err_msg = "{}{}\nUnable to prompt for selection as no tty available. Please update or " \ "install the extension with 'az extension add --upgrade -n <extension-name>'." \ .format(prompt_msg, ext_name) az_error = NoTTYError(tty_err_msg) az_error.print_error() az_error.send_telemetry() parser.exit(exit_code) else: ext_name = ext_name[0] if not ext_name: return # If a valid command has parser error, it may be caused by CLI running on a profile that is # not 'latest' and the command is not supported in that profile. If this command exists in an extension, # CLI will try to download the extension and rerun the command. But the parser will fail again and try to # install the extension and rerun the command infinitely. So we need to check if the latest version of the # extension is already installed and return if yes as the error is not caused by extension not installed. from azure.cli.core.extension import get_extension, ExtensionNotInstalledException from azure.cli.core.extension._resolve import resolve_from_index, NoExtensionCandidatesError try: ext = get_extension(ext_name) except ExtensionNotInstalledException: pass else: try: resolve_from_index(ext_name, cur_version=ext.version, cli_ctx=cli_ctx) except NoExtensionCandidatesError: return telemetry.set_command_details(command_str, parameters=AzCliCommandInvoker._extract_parameter_names(args), # pylint: disable=protected-access extension_name=ext_name) run_after_extension_installed = _get_extension_run_after_dynamic_install_config(cli_ctx) prompt_info = "" if no_prompt: logger.warning('The command requires the extension %s. It will be installed first.', ext_name) install_ext = True else: # yes_prompt from knack.prompting import prompt_y_n, NoTTYException prompt_msg = 'The command requires the extension {}. Do you want to install it now?'.format(ext_name) if run_after_extension_installed: prompt_msg = '{} The command will continue to run after the extension is installed.' \ .format(prompt_msg) NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \ "yes_without_prompt' to allow installing extensions without prompt." try: install_ext = prompt_y_n(prompt_msg, default='y') if install_ext: prompt_info = " with prompt" logger.warning(NO_PROMPT_CONFIG_MSG) except NoTTYException: tty_err_msg = "The command requires the extension {}. " \ "Unable to prompt for extension install confirmation as no tty " \ "available. {}".format(ext_name, NO_PROMPT_CONFIG_MSG) az_error = NoTTYError(tty_err_msg) az_error.print_error() az_error.send_telemetry() parser.exit(exit_code) print_error = True if install_ext: from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name, upgrade=True) if run_after_extension_installed: import subprocess import platform exit_code = subprocess.call(args, shell=platform.system() == 'Windows') # In this case, error msg is for telemetry recording purpose only. # From UX perspective, the command will rerun in subprocess. Whether it succeeds or fails, # mesages will be shown from the subprocess and this process should not print more message to # interrupt that. print_error = False error_msg = ("Extension {} dynamically installed{} and commands will be " "rerun automatically.").format(ext_name, prompt_info) else: error_msg = 'Extension {} installed{}. Please rerun your command.' \ .format(ext_name, prompt_info) else: error_msg = "The command requires the latest version of extension {ext_name}. " \ "To install, run 'az extension add --upgrade -n {ext_name}'.".format( ext_name=ext_name) az_error = CommandNotFoundError(error_msg) if print_error: az_error.print_error() az_error.send_telemetry() parser.exit(exit_code)
def _check_value(self, action, value): # pylint: disable=too-many-statements, too-many-locals # Override to customize the error message when a argument is not among the available choices # converted value must be one of the choices (if specified) if action.choices is not None and value not in action.choices: # pylint: disable=too-many-nested-blocks caused_by_extension_not_installed = False if not self.command_source: candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) error_msg = None # self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'. # use cli_ctx from cli_help which is not lost. cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None) use_dynamic_install = self._get_extension_use_dynamic_install_config( ) if use_dynamic_install != 'no' and not candidates: # Check if the command is from an extension from azure.cli.core.util import roughly_parse_command cmd_list = self.prog.split() + self._raw_arguments command_str = roughly_parse_command(cmd_list[1:]) ext_name = self._search_in_extension_commands(command_str) if ext_name: caused_by_extension_not_installed = True telemetry.set_command_details( command_str, parameters=AzCliCommandInvoker. _extract_parameter_names(cmd_list), # pylint: disable=protected-access extension_name=ext_name) run_after_extension_installed = cli_ctx.config.getboolean( 'extension', 'run_after_dynamic_install', False) if cli_ctx else False if use_dynamic_install == 'yes_without_prompt': logger.warning( 'The command requires the extension %s. ' 'It will be installed first.', ext_name) go_on = True else: from knack.prompting import prompt_y_n, NoTTYException prompt_msg = 'The command requires the extension {}. ' \ 'Do you want to install it now?'.format(ext_name) if run_after_extension_installed: prompt_msg = '{} The command will continue to run after the extension is installed.' \ .format(prompt_msg) NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \ "yes_without_prompt' to allow installing extensions without prompt." try: go_on = prompt_y_n(prompt_msg, default='y') if go_on: logger.warning(NO_PROMPT_CONFIG_MSG) except NoTTYException: logger.warning( "The command requires the extension %s.\n " "Unable to prompt for extension install confirmation as no tty " "available. %s", ext_name, NO_PROMPT_CONFIG_MSG) go_on = False if go_on: from azure.cli.core.extension.operations import add_extension add_extension(cli_ctx=cli_ctx, extension_name=ext_name) if run_after_extension_installed: import subprocess import platform exit_code = subprocess.call( cmd_list, shell=platform.system() == 'Windows') telemetry.set_user_fault( "Extension {} dynamically installed and commands will be " "rerun automatically.".format(ext_name)) self.exit(exit_code) else: error_msg = 'Extension {} installed. Please rerun your command.'.format( ext_name) else: error_msg = "The command requires the extension {ext_name}. " \ "To install, run 'az extension add -n {ext_name}'.".format(ext_name=ext_name) if not error_msg: # parser has no `command_source`, value is part of command itself error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'." \ .format(prog=self.prog, value=value) if use_dynamic_install.lower() == 'no': extensions_link = 'https://docs.microsoft.com/en-us/cli/azure/azure-cli-extensions-overview' error_msg = ( "{msg} " "If the command is from an extension, " "please make sure the corresponding extension is installed. " "To learn more about extensions, please visit " "{extensions_link}").format( msg=error_msg, extensions_link=extensions_link) else: # `command_source` indicates command values have been parsed, value is an argument parameter = action.option_strings[ 0] if action.option_strings else action.dest error_msg = "{prog}: '{value}' is not a valid value for '{param}'. See '{prog} --help'.".format( prog=self.prog, value=value, param=parameter) candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7) telemetry.set_user_fault(error_msg) with CommandLoggerContext(logger): logger.error(error_msg) if not caused_by_extension_not_installed: if candidates: print_args = { 's': 's' if len(candidates) > 1 else '', 'verb': 'are' if len(candidates) > 1 else 'is', 'value': value } self._suggestion_msg.append( "\nThe most similar choice{s} to '{value}' {verb}:". format(**print_args)) self._suggestion_msg.append('\n'.join( ['\t' + candidate for candidate in candidates])) failure_recovery_recommendations = self._get_failure_recovery_recommendations( action) self._suggestion_msg.extend(failure_recovery_recommendations) self._print_suggestion_msg(sys.stderr) self.exit(2)