def main(args):
from azure.cli.core import get_default_cli
from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args
print('Initializing linter with command table and help files...')
# setup CLI to enable command loader
az_cli = get_default_cli()
# load commands, args, and help
create_invoker_and_load_cmds_and_args(az_cli)
loaded_help = get_all_help(az_cli)
command_table = az_cli.invocation.commands_loader.command_table
# format loaded help
loaded_help = {data.command: data for data in loaded_help if data.command}
# load yaml help
help_file_entries = {}
for entry_name, help_yaml in helps.items():
help_entry = yaml.load(help_yaml)
help_file_entries[entry_name] = help_entry
if not args.rule_types_to_run:
args.rule_types_to_run = [
'params', 'commands', 'command_groups', 'help_entries'
]
# find rule exclusions and pass to linter manager
from ..utilities.path import get_command_modules_paths
exclusions = {}
command_modules_paths = get_command_modules_paths()
for _, path in command_modules_paths:
exclusion_path = os.path.join(path, 'linter_exclusions.yml')
if os.path.isfile(exclusion_path):
mod_exclusions = yaml.load(open(exclusion_path))
exclusions.update(mod_exclusions)
# only run linter on modules specified
if args.modules:
from .util import include_mods
command_table, help_file_entries = include_mods(
command_table, help_file_entries, args.modules)
# Instantiate and run Linter
linter_manager = LinterManager(command_table=command_table,
help_file_entries=help_file_entries,
loaded_help=loaded_help,
exclusions=exclusions,
rule_inclusions=args.rules)
exit_code = linter_manager.run(run_params='params'
in args.rule_types_to_run,
run_commands='commands'
in args.rule_types_to_run,
run_command_groups='command_groups'
in args.rule_types_to_run,
run_help_files_entries='help_entries'
in args.rule_types_to_run,
ci=args.ci)
sys.exit(exit_code)
def main(args):
from azure.cli.core import get_default_cli
from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args
print('Initializing linter with command table and help files...')
# setup CLI to enable command loader
az_cli = get_default_cli()
# load commands, args, and help
create_invoker_and_load_cmds_and_args(az_cli)
loaded_help = get_all_help(az_cli)
command_loader = az_cli.invocation.commands_loader
# format loaded help
loaded_help = {data.command: data for data in loaded_help if data.command}
# load yaml help
help_file_entries = {}
for entry_name, help_yaml in helps.items():
help_entry = yaml.load(help_yaml)
help_file_entries[entry_name] = help_entry
if not args.rule_types_to_run:
args.rule_types_to_run = ['params', 'commands', 'command_groups', 'help_entries']
# find rule exclusions and pass to linter manager
from ..utilities.path import get_command_modules_paths, get_extensions_paths
exclusions = {}
command_modules_paths = get_command_modules_paths()
extension_paths = get_extensions_paths()
for gen in (command_modules_paths, extension_paths):
for _, path in gen:
exclusion_path = os.path.join(path, 'linter_exclusions.yml')
if os.path.isfile(exclusion_path):
mod_exclusions = yaml.load(open(exclusion_path))
exclusions.update(mod_exclusions)
# only run linter on modules and extensions specified
if args.modules or args.extensions:
from .util import include_commands
command_loader, help_file_entries = include_commands(
command_loader, help_file_entries, module_inclusions=args.modules, extensions=args.extensions)
# Instantiate and run Linter
linter_manager = LinterManager(command_loader=command_loader,
help_file_entries=help_file_entries,
loaded_help=loaded_help,
exclusions=exclusions,
rule_inclusions=args.rules)
exit_code = linter_manager.run(run_params='params' in args.rule_types_to_run,
run_commands='commands' in args.rule_types_to_run,
run_command_groups='command_groups' in args.rule_types_to_run,
run_help_files_entries='help_entries' in args.rule_types_to_run,
ci=args.ci)
sys.exit(exit_code)
def load_help_files(data):
""" loads all the extra information from help files """
for command_name, help_yaml in helps.items():
help_entry = yaml.load(help_yaml)
try:
help_type = help_entry['type']
except KeyError:
continue
# if there is extra help for this command but it's not reflected in the command table
if command_name not in data and help_type == 'command':
logger.debug('Command: %s not found in command table',
command_name)
continue
short_summary = help_entry.get('short-summary')
short_summary = short_summary() if callable(
short_summary) else short_summary
if short_summary and help_type == 'command':
data[command_name]['help'] = short_summary
else:
# must be a command group or sub-group
data[command_name] = {'help': short_summary}
continue
if 'parameters' in help_entry:
for param in help_entry['parameters']:
# this could fail if the help file and options list are not in the same order
param_name = param['name'].split()[0]
if param_name not in data[command_name]['parameters']:
logger.debug('Command %s does not have parameter: %s',
command_name, param_name)
continue
if 'short-summary' in param:
data[command_name]['parameters'][param_name][
'help'] = param["short-summary"]
if 'examples' in help_entry:
data[command_name]['examples'] = [[
example['name'], example['text']
] for example in help_entry['examples']]
def load_help_files(data):
""" loads all the extra information from help files """
for command_name, help_yaml in helps.items():
help_entry = yaml.safe_load(help_yaml)
try:
help_type = help_entry['type']
except KeyError:
continue
# if there is extra help for this command but it's not reflected in the command table
if command_name not in data and help_type == 'command':
logger.debug('Command: %s not found in command table', command_name)
continue
short_summary = help_entry.get('short-summary')
if short_summary and help_type == 'command':
data[command_name]['help'] = short_summary
else:
# must be a command group or sub-group
data[command_name] = {'help': short_summary}
continue
if 'parameters' in help_entry:
for param in help_entry['parameters']:
# this could fail if the help file and options list are not in the same order
param_name = param['name'].split()[0]
if param_name not in data[command_name]['parameters']:
logger.debug('Command %s does not have parameter: %s', command_name, param_name)
continue
if 'short-summary' in param:
data[command_name]['parameters'][param_name]['help'] = param["short-summary"]
if 'examples' in help_entry:
data[command_name]['examples'] = [[example['name'], example['text']]
for example in help_entry['examples']]
def run_linter(modules=None, rule_types=None, rules=None):
require_azure_cli()
from azure.cli.core import get_default_cli # pylint: disable=import-error
from azure.cli.core.file_util import ( # pylint: disable=import-error
get_all_help, create_invoker_and_load_cmds_and_args)
heading('CLI Linter')
# needed to remove helps from azdev
azdev_helps = helps.copy()
exclusions = {}
selected_modules = get_path_table(include_only=modules)
if not selected_modules:
raise CLIError('No modules selected.')
selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \
list(selected_modules['ext'].keys())
selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \
list(selected_modules['ext'].values())
if selected_mod_names:
display('Modules: {}\n'.format(', '.join(selected_mod_names)))
# collect all rule exclusions
for path in selected_mod_paths:
exclusion_path = os.path.join(path, 'linter_exclusions.yml')
if os.path.isfile(exclusion_path):
mod_exclusions = yaml.load(open(exclusion_path))
exclusions.update(mod_exclusions)
start = time.time()
display('Initializing linter with command table and help files...')
az_cli = get_default_cli()
# load commands, args, and help
create_invoker_and_load_cmds_and_args(az_cli)
loaded_help = get_all_help(az_cli)
stop = time.time()
logger.info('Commands and help loaded in %i sec', stop - start)
command_loader = az_cli.invocation.commands_loader
# format loaded help
loaded_help = {data.command: data for data in loaded_help if data.command}
# load yaml help
help_file_entries = {}
for entry_name, help_yaml in helps.items():
# ignore help entries from azdev itself, unless it also coincides
# with a CLI or extension command name.
if entry_name in azdev_helps and entry_name not in command_loader.command_table:
continue
help_entry = yaml.load(help_yaml)
help_file_entries[entry_name] = help_entry
# trim command table and help to just selected_modules
command_loader, help_file_entries = filter_modules(
command_loader, help_file_entries, modules=selected_mod_names)
if not command_loader.command_table:
raise CLIError('No commands selected to check.')
# Instantiate and run Linter
linter_manager = LinterManager(command_loader=command_loader,
help_file_entries=help_file_entries,
loaded_help=loaded_help,
exclusions=exclusions,
rule_inclusions=rules)
subheading('Results')
logger.info('Running linter: %i commands, %i help entries',
len(command_loader.command_table), len(help_file_entries))
exit_code = linter_manager.run(
run_params=not rule_types or 'params' in rule_types,
run_commands=not rule_types or 'commands' in rule_types,
run_command_groups=not rule_types or 'command_groups'in rule_types,
run_help_files_entries=not rule_types or 'help_entries' in rule_types)
sys.exit(exit_code)
def run_linter(modules=None,
rule_types=None,
rules=None,
ci_exclusions=None,
git_source=None,
git_target=None,
git_repo=None,
include_whl_extensions=False,
min_severity=None,
save_global_exclusion=False):
require_azure_cli()
from azure.cli.core import get_default_cli # pylint: disable=import-error
from azure.cli.core.file_util import ( # pylint: disable=import-error
get_all_help, create_invoker_and_load_cmds_and_args)
heading('CLI Linter')
# allow user to run only on CLI or extensions
cli_only = modules == ['CLI']
ext_only = modules == ['EXT']
if cli_only or ext_only:
modules = None
# process severity option
if min_severity:
try:
min_severity = LinterSeverity.get_linter_severity(min_severity)
except ValueError:
valid_choices = linter_severity_choices()
raise CLIError(
"Please specify a valid linter severity. It should be one of: {}"
.format(", ".join(valid_choices)))
# needed to remove helps from azdev
azdev_helps = helps.copy()
exclusions = {}
selected_modules = get_path_table(
include_only=modules, include_whl_extensions=include_whl_extensions)
if cli_only:
selected_modules['ext'] = {}
if ext_only:
selected_modules['mod'] = {}
selected_modules['core'] = {}
# used to upsert global exclusion
update_global_exclusion = None
if save_global_exclusion and (cli_only or ext_only):
if cli_only:
update_global_exclusion = 'CLI'
if os.path.exists(
os.path.join(get_cli_repo_path(),
'linter_exclusions.yml')):
os.remove(
os.path.join(get_cli_repo_path(), 'linter_exclusions.yml'))
elif ext_only:
update_global_exclusion = 'EXT'
for ext_path in get_ext_repo_paths():
if os.path.exists(
os.path.join(ext_path, 'linter_exclusions.yml')):
os.remove(os.path.join(ext_path, 'linter_exclusions.yml'))
# filter down to only modules that have changed based on git diff
selected_modules = filter_by_git_diff(selected_modules, git_source,
git_target, git_repo)
if not any((selected_modules[x] for x in selected_modules)):
raise CLIError('No modules selected.')
selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \
list(selected_modules['ext'].keys())
selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \
list(selected_modules['ext'].values())
if selected_mod_names:
display('Modules: {}\n'.format(', '.join(selected_mod_names)))
# collect all rule exclusions
for path in selected_mod_paths:
exclusion_path = os.path.join(path, 'linter_exclusions.yml')
if os.path.isfile(exclusion_path):
mod_exclusions = yaml.safe_load(open(exclusion_path))
merge_exclusion(exclusions, mod_exclusions or {})
global_exclusion_paths = [
os.path.join(get_cli_repo_path(), 'linter_exclusions.yml')
]
try:
global_exclusion_paths.extend([
os.path.join(path, 'linter_exclusions.yml')
for path in (get_ext_repo_paths() or [])
])
except CLIError:
pass
for path in global_exclusion_paths:
if os.path.isfile(path):
mod_exclusions = yaml.safe_load(open(path))
merge_exclusion(exclusions, mod_exclusions or {})
start = time.time()
display('Initializing linter with command table and help files...')
az_cli = get_default_cli()
# load commands, args, and help
create_invoker_and_load_cmds_and_args(az_cli)
loaded_help = get_all_help(az_cli)
stop = time.time()
logger.info('Commands and help loaded in %i sec', stop - start)
command_loader = az_cli.invocation.commands_loader
# format loaded help
loaded_help = {data.command: data for data in loaded_help if data.command}
# load yaml help
help_file_entries = {}
for entry_name, help_yaml in helps.items():
# ignore help entries from azdev itself, unless it also coincides
# with a CLI or extension command name.
if entry_name in azdev_helps and entry_name not in command_loader.command_table:
continue
help_entry = yaml.safe_load(help_yaml)
help_file_entries[entry_name] = help_entry
# trim command table and help to just selected_modules
command_loader, help_file_entries = filter_modules(
command_loader,
help_file_entries,
modules=selected_mod_names,
include_whl_extensions=include_whl_extensions)
if not command_loader.command_table:
raise CLIError('No commands selected to check.')
# Instantiate and run Linter
linter_manager = LinterManager(
command_loader=command_loader,
help_file_entries=help_file_entries,
loaded_help=loaded_help,
exclusions=exclusions,
rule_inclusions=rules,
use_ci_exclusions=ci_exclusions,
min_severity=min_severity,
update_global_exclusion=update_global_exclusion)
subheading('Results')
logger.info('Running linter: %i commands, %i help entries',
len(command_loader.command_table), len(help_file_entries))
exit_code = linter_manager.run(run_params=not rule_types
or 'params' in rule_types,
run_commands=not rule_types
or 'commands' in rule_types,
run_command_groups=not rule_types
or 'command_groups' in rule_types,
run_help_files_entries=not rule_types
or 'help_entries' in rule_types)
sys.exit(exit_code)
def main(args):
from azure.cli.core import get_default_cli
from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args
print('Initializing linter with command table and help files...')
# setup CLI to enable command loader
az_cli = get_default_cli()
# load commands, args, and help
create_invoker_and_load_cmds_and_args(az_cli)
loaded_help = get_all_help(az_cli)
command_table = az_cli.invocation.commands_loader.command_table
# format loaded help
loaded_help = {data.command: data for data in loaded_help if data.command}
# load yaml help
help_file_entries = {}
for entry_name, help_yaml in helps.items():
help_entry = yaml.load(help_yaml)
help_file_entries[entry_name] = help_entry
if not args.rule_types_to_run:
args.rule_types_to_run = ['params', 'commands', 'command_groups', 'help_entries']
# find rule exclusions and pass to linter manager
from ..utilities.path import get_command_modules_paths
exclusions = {}
command_modules_paths = get_command_modules_paths()
for _, path in get_command_modules_paths():
exclusion_path = os.path.join(path, 'linter_exclusions.yml')
if os.path.isfile(exclusion_path):
mod_exclusions = yaml.load(open(exclusion_path))
exclusions.update(mod_exclusions)
# only run linter on modules specified
if args.modules:
allowed_module_paths = tuple(path for mod, path in command_modules_paths if mod in args.modules)
for command_name, command in list(command_table.items()):
# brute force way to remove all traces from excluded modules
loader_cls = command.loader.__class__
loader_file_path = inspect.getfile(loader_cls)
if not loader_file_path.startswith(allowed_module_paths):
del command_table[command_name]
help_file_entries.pop(command_name, None)
for group_name in _get_command_groups(command_name):
help_file_entries.pop(group_name, None)
# Instantiate and run Linter
linter_manager = LinterManager(command_table=command_table,
help_file_entries=help_file_entries,
loaded_help=loaded_help,
exclusions=exclusions)
exit_code = linter_manager.run(run_params='params' in args.rule_types_to_run,
run_commands='commands' in args.rule_types_to_run,
run_command_groups='command_groups' in args.rule_types_to_run,
run_help_files_entries='help_entries' in args.rule_types_to_run,
ci=args.ci)
sys.exit(exit_code)