def load_command_table(self, args):
# When executing "azdev linter --include-whl-extensions next" in CI, args is none, so judgment is added
if args:
from azure.cli.core.util import roughly_parse_command
command = roughly_parse_command(args)
if command == 'next' and not self._has_reload_command_table:
self._has_reload_command_table = True
from unittest.mock import patch
with patch.dict("os.environ",
{'AZURE_CORE_USE_COMMAND_INDEX': 'False'}):
self.cli_ctx.invocation.commands_loader.load_command_table(
args)
from azext_next.utils import log_command_history
log_command_history(command, args)
from azext_next.commands import load_command_table
load_command_table(self, args)
return self.command_table
def load_command_table(self, args):
from importlib import import_module
import pkgutil
import traceback
from azure.cli.core.commands import (_load_module_command_loader,
_load_extension_command_loader,
BLOCKED_MODS,
ExtensionCommandSource)
from azure.cli.core.extension import (get_extensions,
get_extension_path,
get_extension_modname)
def _update_command_table_from_modules(args, command_modules=None):
"""Loads command tables from modules and merge into the main command table.
:param args: Arguments of the command.
:param list command_modules: Command modules to load, in the format like ['resource', 'profile'].
If None, will do module discovery and load all modules.
If [], only ALWAYS_LOADED_MODULES will be loaded.
Otherwise, the list will be extended using ALWAYS_LOADED_MODULES.
"""
# As command modules are built-in, the existence of modules in ALWAYS_LOADED_MODULES is NOT checked
if command_modules is not None:
command_modules.extend(ALWAYS_LOADED_MODULES)
else:
# Perform module discovery
command_modules = []
try:
mods_ns_pkg = import_module('azure.cli.command_modules')
command_modules = [
modname for _, modname, _ in pkgutil.iter_modules(
mods_ns_pkg.__path__)
]
logger.debug('Discovered command modules: %s',
command_modules)
except ImportError as e:
logger.warning(e)
count = 0
cumulative_elapsed_time = 0
cumulative_group_count = 0
cumulative_command_count = 0
logger.debug("Loading command modules:")
logger.debug(self.header_mod)
for mod in [m for m in command_modules if m not in BLOCKED_MODS]:
try:
start_time = timeit.default_timer()
module_command_table, module_group_table = _load_module_command_loader(
self, args, mod)
for cmd in module_command_table.values():
cmd.command_source = mod
self.command_table.update(module_command_table)
self.command_group_table.update(module_group_table)
elapsed_time = timeit.default_timer() - start_time
logger.debug(self.item_format_string, mod, elapsed_time,
len(module_group_table),
len(module_command_table))
count += 1
cumulative_elapsed_time += elapsed_time
cumulative_group_count += len(module_group_table)
cumulative_command_count += len(module_command_table)
except Exception as ex: # pylint: disable=broad-except
# Changing this error message requires updating CI script that checks for failed
# module loading.
import azure.cli.core.telemetry as telemetry
logger.error("Error loading command module '%s': %s", mod,
ex)
telemetry.set_exception(
exception=ex,
fault_type='module-load-error-' + mod,
summary='Error loading module: {}'.format(mod))
logger.debug(traceback.format_exc())
# Summary line
logger.debug(self.item_format_string, "Total ({})".format(count),
cumulative_elapsed_time, cumulative_group_count,
cumulative_command_count)
def _update_command_table_from_extensions(ext_suppressions,
extension_modname=None):
"""Loads command tables from extensions and merge into the main command table.
:param ext_suppressions: Extension suppression information.
:param extension_modname: Command modules to load, in the format like ['azext_timeseriesinsights'].
If None, will do extension discovery and load all extensions.
If [], only ALWAYS_LOADED_EXTENSIONS will be loaded.
Otherwise, the list will be extended using ALWAYS_LOADED_EXTENSIONS.
If the extensions in the list are not installed, it will be skipped.
"""
def _handle_extension_suppressions(extensions):
filtered_extensions = []
for ext in extensions:
should_include = True
for suppression in ext_suppressions:
if should_include and suppression.handle_suppress(ext):
should_include = False
if should_include:
filtered_extensions.append(ext)
return filtered_extensions
def _filter_modname(extensions):
# Extension's name may not be the same as its modname. eg. name: virtual-wan, modname: azext_vwan
filtered_extensions = []
for ext in extensions:
ext_mod = get_extension_modname(ext.name, ext.path)
# Filter the extensions according to the index
if ext_mod in extension_modname:
filtered_extensions.append(ext)
extension_modname.remove(ext_mod)
if extension_modname:
logger.debug(
"These extensions are not installed and will be skipped: %s",
extension_modname)
return filtered_extensions
extensions = get_extensions()
if extensions:
if extension_modname is not None:
extension_modname.extend(ALWAYS_LOADED_EXTENSIONS)
extensions = _filter_modname(extensions)
allowed_extensions = _handle_extension_suppressions(extensions)
module_commands = set(self.command_table.keys())
count = 0
cumulative_elapsed_time = 0
cumulative_group_count = 0
cumulative_command_count = 0
logger.debug("Loading extensions:")
logger.debug(self.header_ext)
for ext in allowed_extensions:
try:
# Import in the `for` loop because `allowed_extensions` can be []. In such case we
# don't need to import `check_version_compatibility` at all.
from azure.cli.core.extension.operations import check_version_compatibility
check_version_compatibility(ext.get_metadata())
except CLIError as ex:
# issue warning and skip loading extensions that aren't compatible with the CLI core
logger.warning(ex)
continue
ext_name = ext.name
ext_dir = ext.path or get_extension_path(ext_name)
sys.path.append(ext_dir)
try:
ext_mod = get_extension_modname(ext_name,
ext_dir=ext_dir)
# Add to the map. This needs to happen before we load commands as registering a command
# from an extension requires this map to be up-to-date.
# self._mod_to_ext_map[ext_mod] = ext_name
start_time = timeit.default_timer()
extension_command_table, extension_group_table = \
_load_extension_command_loader(self, args, ext_mod)
for cmd_name, cmd in extension_command_table.items():
cmd.command_source = ExtensionCommandSource(
extension_name=ext_name,
overrides_command=cmd_name in module_commands,
preview=ext.preview,
experimental=ext.experimental)
self.command_table.update(extension_command_table)
self.command_group_table.update(extension_group_table)
elapsed_time = timeit.default_timer() - start_time
logger.debug(self.item_ext_format_string, ext_name,
elapsed_time, len(extension_group_table),
len(extension_command_table), ext_dir)
count += 1
cumulative_elapsed_time += elapsed_time
cumulative_group_count += len(extension_group_table)
cumulative_command_count += len(
extension_command_table)
except Exception as ex: # pylint: disable=broad-except
self.cli_ctx.raise_event(EVENT_FAILED_EXTENSION_LOAD,
extension_name=ext_name)
logger.warning(
"Unable to load extension '%s: %s'. Use --debug for more information.",
ext_name, ex)
logger.debug(traceback.format_exc())
# Summary line
logger.debug(self.item_ext_format_string,
"Total ({})".format(count),
cumulative_elapsed_time, cumulative_group_count,
cumulative_command_count, "")
def _wrap_suppress_extension_func(func, ext):
""" Wrapper method to handle centralization of log messages for extension filters """
res = func(ext)
should_suppress = res
reason = "Use --debug for more information."
if isinstance(res, tuple):
should_suppress, reason = res
suppress_types = (bool, type(None))
if not isinstance(should_suppress, suppress_types):
raise ValueError(
"Command module authoring error: "
"Valid extension suppression values are {} in {}".format(
suppress_types, func))
if should_suppress:
logger.warning("Extension %s (%s) has been suppressed. %s",
ext.name, ext.version, reason)
logger.debug(
"Extension %s (%s) suppressed from being loaded due "
"to %s", ext.name, ext.version, func)
return should_suppress
def _get_extension_suppressions(mod_loaders):
res = []
for m in mod_loaders:
suppressions = getattr(m, 'suppress_extension', None)
if suppressions:
suppressions = suppressions if isinstance(
suppressions, list) else [suppressions]
for sup in suppressions:
if isinstance(sup, ModExtensionSuppress):
res.append(sup)
return res
# Clear the tables to make this method idempotent
self.command_group_table.clear()
self.command_table.clear()
command_index = None
# Set fallback=False to turn off command index in case of regression
use_command_index = self.cli_ctx.config.getboolean('core',
'use_command_index',
fallback=True)
if use_command_index:
command_index = CommandIndex(self.cli_ctx)
index_result = command_index.get(args)
if index_result:
index_modules, index_extensions = index_result
# Always load modules and extensions, because some of them (like those in
# ALWAYS_LOADED_EXTENSIONS) don't expose a command, but hooks into handlers in CLI core
_update_command_table_from_modules(args, index_modules)
# The index won't contain suppressed extensions
_update_command_table_from_extensions([], index_extensions)
logger.debug("Loaded %d groups, %d commands.",
len(self.command_group_table),
len(self.command_table))
from azure.cli.core.util import roughly_parse_command
# The index may be outdated. Make sure the command appears in the loaded command table
raw_cmd = roughly_parse_command(args)
for cmd in self.command_table:
if raw_cmd.startswith(cmd):
# For commands with positional arguments, the raw command won't match the one in the
# command table. For example, `az find vm create` won't exist in the command table, but the
# corresponding command should be `az find`.
# raw command : az find vm create
# command table: az find
# remaining : vm create
logger.debug("Found a match in the command table.")
logger.debug("Raw command : %s", raw_cmd)
logger.debug("Command table: %s", cmd)
remaining = raw_cmd[len(cmd) + 1:]
if remaining:
logger.debug("remaining : %s %s",
' ' * len(cmd), remaining)
return self.command_table
# For command group, it must be an exact match, as no positional argument is supported by
# command group operations.
if raw_cmd in self.command_group_table:
logger.debug(
"Found a match in the command group table for '%s'.",
raw_cmd)
return self.command_table
logger.debug(
"Could not find a match in the command or command group table for '%s'. "
"The index may be outdated.", raw_cmd)
else:
logger.debug("No module found from index for '%s'", args)
# No module found from the index. Load all command modules and extensions
logger.debug("Loading all modules and extensions")
_update_command_table_from_modules(args)
ext_suppressions = _get_extension_suppressions(self.loaders)
# We always load extensions even if the appropriate module has been loaded
# as an extension could override the commands already loaded.
_update_command_table_from_extensions(ext_suppressions)
logger.debug("Loaded %d groups, %d commands.",
len(self.command_group_table), len(self.command_table))
if use_command_index:
command_index.update(self.command_table)
return self.command_table
def _check_value(self, action, value): # pylint: disable=too-many-statements, too-many-locals
# Override to customize the error message when a argument is not among the available choices
# converted value must be one of the choices (if specified)
if action.choices is not None and value not in action.choices: # pylint: disable=too-many-nested-blocks
# self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'.
# use cli_ctx from cli_help which is not lost.
cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx if self.cli_help else None)
caused_by_extension_not_installed = False
command_name_inferred = self.prog
error_msg = None
if not self.command_source:
candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7)
if candidates:
# use the most likely candidate to replace the misspelled command
args = self.prog.split() + self._raw_arguments
args_inferred = [item if item != value else candidates[0] for item in args]
command_name_inferred = ' '.join(args_inferred).split('-')[0]
use_dynamic_install = self._get_extension_use_dynamic_install_config()
if use_dynamic_install != 'no' and not candidates:
# Check if the command is from an extension
from azure.cli.core.util import roughly_parse_command
cmd_list = self.prog.split() + self._raw_arguments
command_str = roughly_parse_command(cmd_list[1:])
ext_name = self._search_in_extension_commands(command_str)
if ext_name:
caused_by_extension_not_installed = True
telemetry.set_command_details(command_str,
parameters=AzCliCommandInvoker._extract_parameter_names(cmd_list), # pylint: disable=protected-access
extension_name=ext_name)
run_after_extension_installed = self._get_extension_run_after_dynamic_install_config()
if use_dynamic_install == 'yes_without_prompt':
logger.warning('The command requires the extension %s. '
'It will be installed first.', ext_name)
go_on = True
else:
from knack.prompting import prompt_y_n, NoTTYException
prompt_msg = 'The command requires the extension {}. ' \
'Do you want to install it now?'.format(ext_name)
if run_after_extension_installed:
prompt_msg = '{} The command will continue to run after the extension is installed.' \
.format(prompt_msg)
NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \
"yes_without_prompt' to allow installing extensions without prompt."
try:
go_on = prompt_y_n(prompt_msg, default='y')
if go_on:
logger.warning(NO_PROMPT_CONFIG_MSG)
except NoTTYException:
logger.warning("The command requires the extension %s.\n "
"Unable to prompt for extension install confirmation as no tty "
"available. %s", ext_name, NO_PROMPT_CONFIG_MSG)
go_on = False
if go_on:
from azure.cli.core.extension.operations import add_extension
add_extension(cli_ctx=cli_ctx, extension_name=ext_name, upgrade=True)
if run_after_extension_installed:
import subprocess
import platform
exit_code = subprocess.call(cmd_list, shell=platform.system() == 'Windows')
error_msg = ("Extension {} dynamically installed and commands will be "
"rerun automatically.").format(ext_name)
telemetry.set_user_fault(error_msg)
self.exit(exit_code)
else:
with CommandLoggerContext(logger):
error_msg = 'Extension {} installed. Please rerun your command.'.format(ext_name)
logger.error(error_msg)
telemetry.set_user_fault(error_msg)
self.exit(2)
else:
error_msg = "The command requires the latest version of extension {ext_name}. " \
"To install, run 'az extension add --upgrade -n {ext_name}'.".format(ext_name=ext_name)
if not error_msg:
# parser has no `command_source`, value is part of command itself
error_msg = "'{value}' is misspelled or not recognized by the system.".format(value=value)
az_error = CommandNotFoundError(error_msg)
else:
# `command_source` indicates command values have been parsed, value is an argument
parameter = action.option_strings[0] if action.option_strings else action.dest
error_msg = "{prog}: '{value}' is not a valid value for '{param}'.".format(
prog=self.prog, value=value, param=parameter)
candidates = difflib.get_close_matches(value, action.choices, cutoff=0.7)
az_error = InvalidArgumentValueError(error_msg)
command_arguments = self._get_failure_recovery_arguments(action)
if candidates:
az_error.set_recommendation("Did you mean '{}' ?".format(candidates[0]))
# recommend a command for user
recommender = CommandRecommender(*command_arguments, error_msg, cli_ctx)
recommender.set_help_examples(self.get_examples(command_name_inferred))
recommended_command = recommender.recommend_a_command()
if recommended_command:
az_error.set_recommendation("Try this: '{}'".format(recommended_command))
# remind user to check extensions if we can not find a command to recommend
if isinstance(az_error, CommandNotFoundError) \
and not az_error.recommendations and self.prog == 'az' \
and use_dynamic_install == 'no':
az_error.set_recommendation(EXTENSION_REFERENCE)
az_error.set_recommendation(OVERVIEW_REFERENCE.format(command=self.prog))
if not caused_by_extension_not_installed:
az_error.print_error()
az_error.send_telemetry()
self.exit(2)
def _check_value_in_extensions(cli_ctx, parser, args, no_prompt): # pylint: disable=too-many-statements, too-many-locals
"""Check if the command args can be found in extension commands.
Exit command if the error is caused by an extension not installed.
Otherwise return.
"""
# Check if the command is from an extension
from azure.cli.core.util import roughly_parse_command
from azure.cli.core.azclierror import NoTTYError
exit_code = 2
command_str = roughly_parse_command(args[1:])
allow_prefix_match = args[-1] == '-h' or args[-1] == '--help'
ext_name = _search_in_extension_commands(cli_ctx, command_str, allow_prefix_match=allow_prefix_match)
# ext_name is a list if the input command matches the prefix of one or more extension commands,
# for instance: `az blueprint` when running `az blueprint -h`
# ext_name is a str if the input command matches a complete command of an extension,
# for instance: `az blueprint create`
if isinstance(ext_name, list):
if len(ext_name) > 1:
from knack.prompting import prompt_choice_list, NoTTYException
prompt_msg = "The command requires the latest version of one of the following " \
"extensions. You need to pick one to install:"
try:
choice_idx = prompt_choice_list(prompt_msg, ext_name)
ext_name = ext_name[choice_idx]
no_prompt = True
except NoTTYException:
tty_err_msg = "{}{}\nUnable to prompt for selection as no tty available. Please update or " \
"install the extension with 'az extension add --upgrade -n <extension-name>'." \
.format(prompt_msg, ext_name)
az_error = NoTTYError(tty_err_msg)
az_error.print_error()
az_error.send_telemetry()
parser.exit(exit_code)
else:
ext_name = ext_name[0]
if not ext_name:
return
# If a valid command has parser error, it may be caused by CLI running on a profile that is
# not 'latest' and the command is not supported in that profile. If this command exists in an extension,
# CLI will try to download the extension and rerun the command. But the parser will fail again and try to
# install the extension and rerun the command infinitely. So we need to check if the latest version of the
# extension is already installed and return if yes as the error is not caused by extension not installed.
from azure.cli.core.extension import get_extension, ExtensionNotInstalledException
from azure.cli.core.extension._resolve import resolve_from_index, NoExtensionCandidatesError
try:
ext = get_extension(ext_name)
except ExtensionNotInstalledException:
pass
else:
try:
resolve_from_index(ext_name, cur_version=ext.version, cli_ctx=cli_ctx)
except NoExtensionCandidatesError:
return
telemetry.set_command_details(command_str,
parameters=AzCliCommandInvoker._extract_parameter_names(args), # pylint: disable=protected-access
extension_name=ext_name)
run_after_extension_installed = _get_extension_run_after_dynamic_install_config(cli_ctx)
prompt_info = ""
if no_prompt:
logger.warning('The command requires the extension %s. It will be installed first.', ext_name)
install_ext = True
else: # yes_prompt
from knack.prompting import prompt_y_n, NoTTYException
prompt_msg = 'The command requires the extension {}. Do you want to install it now?'.format(ext_name)
if run_after_extension_installed:
prompt_msg = '{} The command will continue to run after the extension is installed.' \
.format(prompt_msg)
NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \
"yes_without_prompt' to allow installing extensions without prompt."
try:
install_ext = prompt_y_n(prompt_msg, default='y')
if install_ext:
prompt_info = " with prompt"
logger.warning(NO_PROMPT_CONFIG_MSG)
except NoTTYException:
tty_err_msg = "The command requires the extension {}. " \
"Unable to prompt for extension install confirmation as no tty " \
"available. {}".format(ext_name, NO_PROMPT_CONFIG_MSG)
az_error = NoTTYError(tty_err_msg)
az_error.print_error()
az_error.send_telemetry()
parser.exit(exit_code)
print_error = True
if install_ext:
from azure.cli.core.extension.operations import add_extension
add_extension(cli_ctx=cli_ctx, extension_name=ext_name, upgrade=True)
if run_after_extension_installed:
import subprocess
import platform
exit_code = subprocess.call(args, shell=platform.system() == 'Windows')
# In this case, error msg is for telemetry recording purpose only.
# From UX perspective, the command will rerun in subprocess. Whether it succeeds or fails,
# mesages will be shown from the subprocess and this process should not print more message to
# interrupt that.
print_error = False
error_msg = ("Extension {} dynamically installed{} and commands will be "
"rerun automatically.").format(ext_name, prompt_info)
else:
error_msg = 'Extension {} installed{}. Please rerun your command.' \
.format(ext_name, prompt_info)
else:
error_msg = "The command requires the latest version of extension {ext_name}. " \
"To install, run 'az extension add --upgrade -n {ext_name}'.".format(
ext_name=ext_name)
az_error = CommandNotFoundError(error_msg)
if print_error:
az_error.print_error()
az_error.send_telemetry()
parser.exit(exit_code)
def _check_value(self, action, value): # pylint: disable=too-many-statements, too-many-locals
# Override to customize the error message when a argument is not among the available choices
# converted value must be one of the choices (if specified)
if action.choices is not None and value not in action.choices: # pylint: disable=too-many-nested-blocks
caused_by_extension_not_installed = False
if not self.command_source:
candidates = difflib.get_close_matches(value,
action.choices,
cutoff=0.7)
error_msg = None
# self.cli_ctx is None when self.prog is beyond 'az', such as 'az iot'.
# use cli_ctx from cli_help which is not lost.
cli_ctx = self.cli_ctx or (self.cli_help.cli_ctx
if self.cli_help else None)
use_dynamic_install = self._get_extension_use_dynamic_install_config(
)
if use_dynamic_install != 'no' and not candidates:
# Check if the command is from an extension
from azure.cli.core.util import roughly_parse_command
cmd_list = self.prog.split() + self._raw_arguments
command_str = roughly_parse_command(cmd_list[1:])
ext_name = self._search_in_extension_commands(command_str)
if ext_name:
caused_by_extension_not_installed = True
telemetry.set_command_details(
command_str,
parameters=AzCliCommandInvoker.
_extract_parameter_names(cmd_list), # pylint: disable=protected-access
extension_name=ext_name)
run_after_extension_installed = cli_ctx.config.getboolean(
'extension', 'run_after_dynamic_install',
False) if cli_ctx else False
if use_dynamic_install == 'yes_without_prompt':
logger.warning(
'The command requires the extension %s. '
'It will be installed first.', ext_name)
go_on = True
else:
from knack.prompting import prompt_y_n, NoTTYException
prompt_msg = 'The command requires the extension {}. ' \
'Do you want to install it now?'.format(ext_name)
if run_after_extension_installed:
prompt_msg = '{} The command will continue to run after the extension is installed.' \
.format(prompt_msg)
NO_PROMPT_CONFIG_MSG = "Run 'az config set extension.use_dynamic_install=" \
"yes_without_prompt' to allow installing extensions without prompt."
try:
go_on = prompt_y_n(prompt_msg, default='y')
if go_on:
logger.warning(NO_PROMPT_CONFIG_MSG)
except NoTTYException:
logger.warning(
"The command requires the extension %s.\n "
"Unable to prompt for extension install confirmation as no tty "
"available. %s", ext_name,
NO_PROMPT_CONFIG_MSG)
go_on = False
if go_on:
from azure.cli.core.extension.operations import add_extension
add_extension(cli_ctx=cli_ctx,
extension_name=ext_name)
if run_after_extension_installed:
import subprocess
import platform
exit_code = subprocess.call(
cmd_list,
shell=platform.system() == 'Windows')
telemetry.set_user_fault(
"Extension {} dynamically installed and commands will be "
"rerun automatically.".format(ext_name))
self.exit(exit_code)
else:
error_msg = 'Extension {} installed. Please rerun your command.'.format(
ext_name)
else:
error_msg = "The command requires the extension {ext_name}. " \
"To install, run 'az extension add -n {ext_name}'.".format(ext_name=ext_name)
if not error_msg:
# parser has no `command_source`, value is part of command itself
error_msg = "{prog}: '{value}' is not in the '{prog}' command group. See '{prog} --help'." \
.format(prog=self.prog, value=value)
if use_dynamic_install.lower() == 'no':
extensions_link = 'https://docs.microsoft.com/en-us/cli/azure/azure-cli-extensions-overview'
error_msg = (
"{msg} "
"If the command is from an extension, "
"please make sure the corresponding extension is installed. "
"To learn more about extensions, please visit "
"{extensions_link}").format(
msg=error_msg, extensions_link=extensions_link)
else:
# `command_source` indicates command values have been parsed, value is an argument
parameter = action.option_strings[
0] if action.option_strings else action.dest
error_msg = "{prog}: '{value}' is not a valid value for '{param}'. See '{prog} --help'.".format(
prog=self.prog, value=value, param=parameter)
candidates = difflib.get_close_matches(value,
action.choices,
cutoff=0.7)
telemetry.set_user_fault(error_msg)
with CommandLoggerContext(logger):
logger.error(error_msg)
if not caused_by_extension_not_installed:
if candidates:
print_args = {
's': 's' if len(candidates) > 1 else '',
'verb': 'are' if len(candidates) > 1 else 'is',
'value': value
}
self._suggestion_msg.append(
"\nThe most similar choice{s} to '{value}' {verb}:".
format(**print_args))
self._suggestion_msg.append('\n'.join(
['\t' + candidate for candidate in candidates]))
failure_recovery_recommendations = self._get_failure_recovery_recommendations(
action)
self._suggestion_msg.extend(failure_recovery_recommendations)
self._print_suggestion_msg(sys.stderr)
self.exit(2)
