def create_invoker_and_load_cmds_and_args(cli_ctx):
global loaded_helps
from knack import events
from azure.cli.core.commands import register_cache_arguments
from azure.cli.core.commands.arm import register_global_subscription_argument, register_ids_argument
invoker = cli_ctx.invocation_cls(cli_ctx=cli_ctx, commands_loader_cls=cli_ctx.commands_loader_cls,
parser_cls=cli_ctx.parser_cls, help_cls=cli_ctx.help_cls)
cli_ctx.invocation = invoker
invoker.commands_loader.skip_applicability = True
temp_help = helps.copy()
invoker.commands_loader.load_command_table(None) # this ends up loading all the helpfiles, which could be problematic with duplicate key commands like acs create
helps.clear()
helps.update(temp_help)
# turn off applicability check for applicable loaders
new_cmd_to_loader_map = {}
new_command_group_table = {}
new_command_table = {}
for cmd in loaded_helps.keys():
if cmd in invoker.commands_loader.cmd_to_loader_map: # if a command from help, then update commands_loader
new_cmd_to_loader_map[cmd] = invoker.commands_loader.cmd_to_loader_map[cmd]
new_command_table[cmd] = invoker.commands_loader.command_table[cmd]
else: # else a group then update
new_command_group_table[cmd] = invoker.commands_loader.command_group_table[cmd]
# include commands of groups in help.py, even though the commands themselves might not be in help.py, so their subparsers can be added.
for cmd in new_command_group_table.keys():
all_cmds = invoker.commands_loader.cmd_to_loader_map.keys()
for old_cmd in all_cmds:
if old_cmd.startswith(cmd) and old_cmd != cmd:
new_cmd_to_loader_map[old_cmd] = invoker.commands_loader.cmd_to_loader_map[old_cmd]
new_command_table[old_cmd] = invoker.commands_loader.command_table[old_cmd]
invoker.commands_loader.cmd_to_loader_map = new_cmd_to_loader_map
invoker.commands_loader.command_table = new_command_table
invoker.commands_loader.command_group_table = new_command_group_table
for loaders in invoker.commands_loader.cmd_to_loader_map.values():
for loader in loaders:
loader.skip_applicability = True
for command in invoker.commands_loader.command_table:
invoker.commands_loader.load_arguments(command)
assert len(new_command_table) == len(new_cmd_to_loader_map)
assert set(list(new_command_table.keys()) + list(new_command_group_table.keys())) >= set(loaded_helps.keys())
register_global_subscription_argument(cli_ctx)
register_ids_argument(cli_ctx) # global subscription must be registered first!
register_cache_arguments(cli_ctx)
cli_ctx.raise_event(events.EVENT_INVOKER_POST_CMD_TBL_CREATE, commands_loader=invoker.commands_loader)
invoker.parser.load_command_table(invoker.commands_loader)
def create_invoker_and_load_cmds_and_args(cli_ctx):
global loaded_helps
from knack import events
from azure.cli.core.commands import register_cache_arguments
from azure.cli.core.commands.arm import register_global_subscription_argument, register_ids_argument
invoker = cli_ctx.invocation_cls(
cli_ctx=cli_ctx,
commands_loader_cls=cli_ctx.commands_loader_cls,
parser_cls=cli_ctx.parser_cls,
help_cls=cli_ctx.help_cls)
cli_ctx.invocation = invoker
invoker.commands_loader.skip_applicability = True
temp_help = helps.copy()
invoker.commands_loader.load_command_table(
None
) # this ends up loading all the helpfiles, which could be problematic with duplicate key commands like acs create
helps.clear()
helps.update(temp_help)
# turn off applicability check for applicable loaders
new_cmd_to_loader_map = {}
new_command_group_table = {}
new_command_table = {}
for cmd in loaded_helps.keys():
if cmd in invoker.commands_loader.cmd_to_loader_map: # if a command from help, then update commands_loader
new_cmd_to_loader_map[
cmd] = invoker.commands_loader.cmd_to_loader_map[cmd]
new_command_table[cmd] = invoker.commands_loader.command_table[cmd]
else: # else a group then update
new_command_group_table[
cmd] = invoker.commands_loader.command_group_table[cmd]
# include commands of groups in help.py, even though the commands themselves might not be in help.py, so their subparsers can be added.
for cmd in new_command_group_table.keys():
all_cmds = invoker.commands_loader.cmd_to_loader_map.keys()
for old_cmd in all_cmds:
if old_cmd.startswith(cmd) and old_cmd != cmd:
new_cmd_to_loader_map[
old_cmd] = invoker.commands_loader.cmd_to_loader_map[
old_cmd]
new_command_table[
old_cmd] = invoker.commands_loader.command_table[old_cmd]
invoker.commands_loader.cmd_to_loader_map = new_cmd_to_loader_map
invoker.commands_loader.command_table = new_command_table
invoker.commands_loader.command_group_table = new_command_group_table
for loaders in invoker.commands_loader.cmd_to_loader_map.values():
for loader in loaders:
loader.skip_applicability = True
for command in invoker.commands_loader.command_table:
invoker.commands_loader.load_arguments(command)
assert len(new_command_table) == len(new_cmd_to_loader_map)
assert set(
list(new_command_table.keys()) +
list(new_command_group_table.keys())) >= set(loaded_helps.keys())
register_global_subscription_argument(cli_ctx)
register_ids_argument(
cli_ctx) # global subscription must be registered first!
register_cache_arguments(cli_ctx)
cli_ctx.raise_event(events.EVENT_INVOKER_POST_CMD_TBL_CREATE,
commands_loader=invoker.commands_loader)
invoker.parser.load_command_table(invoker.commands_loader)
def run_linter(modules=None, rule_types=None, rules=None):
require_azure_cli()
from azure.cli.core import get_default_cli # pylint: disable=import-error
from azure.cli.core.file_util import ( # pylint: disable=import-error
get_all_help, create_invoker_and_load_cmds_and_args)
heading('CLI Linter')
# needed to remove helps from azdev
azdev_helps = helps.copy()
exclusions = {}
selected_modules = get_path_table(include_only=modules)
if not selected_modules:
raise CLIError('No modules selected.')
selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \
list(selected_modules['ext'].keys())
selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \
list(selected_modules['ext'].values())
if selected_mod_names:
display('Modules: {}\n'.format(', '.join(selected_mod_names)))
# collect all rule exclusions
for path in selected_mod_paths:
exclusion_path = os.path.join(path, 'linter_exclusions.yml')
if os.path.isfile(exclusion_path):
mod_exclusions = yaml.load(open(exclusion_path))
exclusions.update(mod_exclusions)
start = time.time()
display('Initializing linter with command table and help files...')
az_cli = get_default_cli()
# load commands, args, and help
create_invoker_and_load_cmds_and_args(az_cli)
loaded_help = get_all_help(az_cli)
stop = time.time()
logger.info('Commands and help loaded in %i sec', stop - start)
command_loader = az_cli.invocation.commands_loader
# format loaded help
loaded_help = {data.command: data for data in loaded_help if data.command}
# load yaml help
help_file_entries = {}
for entry_name, help_yaml in helps.items():
# ignore help entries from azdev itself, unless it also coincides
# with a CLI or extension command name.
if entry_name in azdev_helps and entry_name not in command_loader.command_table:
continue
help_entry = yaml.load(help_yaml)
help_file_entries[entry_name] = help_entry
# trim command table and help to just selected_modules
command_loader, help_file_entries = filter_modules(
command_loader, help_file_entries, modules=selected_mod_names)
if not command_loader.command_table:
raise CLIError('No commands selected to check.')
# Instantiate and run Linter
linter_manager = LinterManager(command_loader=command_loader,
help_file_entries=help_file_entries,
loaded_help=loaded_help,
exclusions=exclusions,
rule_inclusions=rules)
subheading('Results')
logger.info('Running linter: %i commands, %i help entries',
len(command_loader.command_table), len(help_file_entries))
exit_code = linter_manager.run(
run_params=not rule_types or 'params' in rule_types,
run_commands=not rule_types or 'commands' in rule_types,
run_command_groups=not rule_types or 'command_groups'in rule_types,
run_help_files_entries=not rule_types or 'help_entries' in rule_types)
sys.exit(exit_code)
def run_linter(modules=None,
rule_types=None,
rules=None,
ci_exclusions=None,
git_source=None,
git_target=None,
git_repo=None,
include_whl_extensions=False,
min_severity=None,
save_global_exclusion=False):
require_azure_cli()
from azure.cli.core import get_default_cli # pylint: disable=import-error
from azure.cli.core.file_util import ( # pylint: disable=import-error
get_all_help, create_invoker_and_load_cmds_and_args)
heading('CLI Linter')
# allow user to run only on CLI or extensions
cli_only = modules == ['CLI']
ext_only = modules == ['EXT']
if cli_only or ext_only:
modules = None
# process severity option
if min_severity:
try:
min_severity = LinterSeverity.get_linter_severity(min_severity)
except ValueError:
valid_choices = linter_severity_choices()
raise CLIError(
"Please specify a valid linter severity. It should be one of: {}"
.format(", ".join(valid_choices)))
# needed to remove helps from azdev
azdev_helps = helps.copy()
exclusions = {}
selected_modules = get_path_table(
include_only=modules, include_whl_extensions=include_whl_extensions)
if cli_only:
selected_modules['ext'] = {}
if ext_only:
selected_modules['mod'] = {}
selected_modules['core'] = {}
# used to upsert global exclusion
update_global_exclusion = None
if save_global_exclusion and (cli_only or ext_only):
if cli_only:
update_global_exclusion = 'CLI'
if os.path.exists(
os.path.join(get_cli_repo_path(),
'linter_exclusions.yml')):
os.remove(
os.path.join(get_cli_repo_path(), 'linter_exclusions.yml'))
elif ext_only:
update_global_exclusion = 'EXT'
for ext_path in get_ext_repo_paths():
if os.path.exists(
os.path.join(ext_path, 'linter_exclusions.yml')):
os.remove(os.path.join(ext_path, 'linter_exclusions.yml'))
# filter down to only modules that have changed based on git diff
selected_modules = filter_by_git_diff(selected_modules, git_source,
git_target, git_repo)
if not any((selected_modules[x] for x in selected_modules)):
raise CLIError('No modules selected.')
selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \
list(selected_modules['ext'].keys())
selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \
list(selected_modules['ext'].values())
if selected_mod_names:
display('Modules: {}\n'.format(', '.join(selected_mod_names)))
# collect all rule exclusions
for path in selected_mod_paths:
exclusion_path = os.path.join(path, 'linter_exclusions.yml')
if os.path.isfile(exclusion_path):
mod_exclusions = yaml.safe_load(open(exclusion_path))
merge_exclusion(exclusions, mod_exclusions or {})
global_exclusion_paths = [
os.path.join(get_cli_repo_path(), 'linter_exclusions.yml')
]
try:
global_exclusion_paths.extend([
os.path.join(path, 'linter_exclusions.yml')
for path in (get_ext_repo_paths() or [])
])
except CLIError:
pass
for path in global_exclusion_paths:
if os.path.isfile(path):
mod_exclusions = yaml.safe_load(open(path))
merge_exclusion(exclusions, mod_exclusions or {})
start = time.time()
display('Initializing linter with command table and help files...')
az_cli = get_default_cli()
# load commands, args, and help
create_invoker_and_load_cmds_and_args(az_cli)
loaded_help = get_all_help(az_cli)
stop = time.time()
logger.info('Commands and help loaded in %i sec', stop - start)
command_loader = az_cli.invocation.commands_loader
# format loaded help
loaded_help = {data.command: data for data in loaded_help if data.command}
# load yaml help
help_file_entries = {}
for entry_name, help_yaml in helps.items():
# ignore help entries from azdev itself, unless it also coincides
# with a CLI or extension command name.
if entry_name in azdev_helps and entry_name not in command_loader.command_table:
continue
help_entry = yaml.safe_load(help_yaml)
help_file_entries[entry_name] = help_entry
# trim command table and help to just selected_modules
command_loader, help_file_entries = filter_modules(
command_loader,
help_file_entries,
modules=selected_mod_names,
include_whl_extensions=include_whl_extensions)
if not command_loader.command_table:
raise CLIError('No commands selected to check.')
# Instantiate and run Linter
linter_manager = LinterManager(
command_loader=command_loader,
help_file_entries=help_file_entries,
loaded_help=loaded_help,
exclusions=exclusions,
rule_inclusions=rules,
use_ci_exclusions=ci_exclusions,
min_severity=min_severity,
update_global_exclusion=update_global_exclusion)
subheading('Results')
logger.info('Running linter: %i commands, %i help entries',
len(command_loader.command_table), len(help_file_entries))
exit_code = linter_manager.run(run_params=not rule_types
or 'params' in rule_types,
run_commands=not rule_types
or 'commands' in rule_types,
run_command_groups=not rule_types
or 'command_groups' in rule_types,
run_help_files_entries=not rule_types
or 'help_entries' in rule_types)
sys.exit(exit_code)
