def check_cli(cmd): from azure.cli.core.file_util import ( create_invoker_and_load_cmds_and_args, get_all_help) exceptions = {} print('Running CLI self-test.\n') print('Loading all commands and arguments...') try: create_invoker_and_load_cmds_and_args(cmd.cli_ctx) print('Commands loaded OK.\n') except Exception as ex: # pylint: disable=broad-except exceptions['load_commands'] = ex logger.error('Error occurred loading commands!\n') print('Retrieving all help...') try: get_all_help(cmd.cli_ctx, skip=False) print('Help loaded OK.\n') except Exception as ex: # pylint: disable=broad-except exceptions['load_help'] = ex logger.error('Error occurred loading help!\n') if not exceptions: print('CLI self-test completed: OK') else: raise CLIError(exceptions)
def main(args): from azure.cli.core import get_default_cli from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args print('Initializing linter with command table and help files...') # setup CLI to enable command loader az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) command_table = az_cli.invocation.commands_loader.command_table # format loaded help loaded_help = {data.command: data for data in loaded_help if data.command} # load yaml help help_file_entries = {} for entry_name, help_yaml in helps.items(): help_entry = yaml.load(help_yaml) help_file_entries[entry_name] = help_entry if not args.rule_types_to_run: args.rule_types_to_run = [ 'params', 'commands', 'command_groups', 'help_entries' ] # find rule exclusions and pass to linter manager from ..utilities.path import get_command_modules_paths exclusions = {} command_modules_paths = get_command_modules_paths() for _, path in command_modules_paths: exclusion_path = os.path.join(path, 'linter_exclusions.yml') if os.path.isfile(exclusion_path): mod_exclusions = yaml.load(open(exclusion_path)) exclusions.update(mod_exclusions) # only run linter on modules specified if args.modules: from .util import include_mods command_table, help_file_entries = include_mods( command_table, help_file_entries, args.modules) # Instantiate and run Linter linter_manager = LinterManager(command_table=command_table, help_file_entries=help_file_entries, loaded_help=loaded_help, exclusions=exclusions, rule_inclusions=args.rules) exit_code = linter_manager.run(run_params='params' in args.rule_types_to_run, run_commands='commands' in args.rule_types_to_run, run_command_groups='command_groups' in args.rule_types_to_run, run_help_files_entries='help_entries' in args.rule_types_to_run, ci=args.ci) sys.exit(exit_code)
def check_cli(cmd): from azure.cli.core.file_util import ( create_invoker_and_load_cmds_and_args, get_all_help) exceptions = {} print('Running CLI self-test.\n') print('Loading all commands and arguments...') try: create_invoker_and_load_cmds_and_args(cmd.cli_ctx) print('Commands loaded OK.\n') except Exception as ex: # pylint: disable=broad-except exceptions['load_commands'] = ex logger.error('Error occurred loading commands!\n') print('Retrieving all help...') try: get_all_help(cmd.cli_ctx) print('Help loaded OK.\n') except Exception as ex: # pylint: disable=broad-except exceptions['load_help'] = ex logger.error('Error occurred loading help!\n') if not exceptions: print('CLI self-test completed: OK') else: raise CLIError(exceptions)
def main(args): from azure.cli.core import get_default_cli from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args print('Initializing linter with command table and help files...') # setup CLI to enable command loader az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) command_loader = az_cli.invocation.commands_loader # format loaded help loaded_help = {data.command: data for data in loaded_help if data.command} # load yaml help help_file_entries = {} for entry_name, help_yaml in helps.items(): help_entry = yaml.load(help_yaml) help_file_entries[entry_name] = help_entry if not args.rule_types_to_run: args.rule_types_to_run = ['params', 'commands', 'command_groups', 'help_entries'] # find rule exclusions and pass to linter manager from ..utilities.path import get_command_modules_paths, get_extensions_paths exclusions = {} command_modules_paths = get_command_modules_paths() extension_paths = get_extensions_paths() for gen in (command_modules_paths, extension_paths): for _, path in gen: exclusion_path = os.path.join(path, 'linter_exclusions.yml') if os.path.isfile(exclusion_path): mod_exclusions = yaml.load(open(exclusion_path)) exclusions.update(mod_exclusions) # only run linter on modules and extensions specified if args.modules or args.extensions: from .util import include_commands command_loader, help_file_entries = include_commands( command_loader, help_file_entries, module_inclusions=args.modules, extensions=args.extensions) # Instantiate and run Linter linter_manager = LinterManager(command_loader=command_loader, help_file_entries=help_file_entries, loaded_help=loaded_help, exclusions=exclusions, rule_inclusions=args.rules) exit_code = linter_manager.run(run_params='params' in args.rule_types_to_run, run_commands='commands' in args.rule_types_to_run, run_command_groups='command_groups' in args.rule_types_to_run, run_help_files_entries='help_entries' in args.rule_types_to_run, ci=args.ci) sys.exit(exit_code)
def verify_load_all(_): from azure.cli.core import get_default_cli from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args print('Loading all commands, arguments, and help...') # setup CLI to enable command loader az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) print('Everything loaded successfully.')
def _benchmark_load_all_commands(): try: from azure.cli.core import get_default_cli from azure.cli.core.file_util import create_invoker_and_load_cmds_and_args except ImportError: raise CLIError("Azure CLI is not installed") az_cli = get_default_cli() create_invoker_and_load_cmds_and_args(az_cli) commands = list(az_cli.invocation.commands_loader.command_table.keys()) commands = [cmd + " --help" for cmd in commands] return sorted(commands)
def verify_load_all(_): from azure.cli.core import get_default_cli, EVENT_FAILED_EXTENSION_LOAD from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args print('Loading all commands, arguments, and help...') # setup CLI to enable command loader and register event az_cli = get_default_cli() az_cli.register_event(EVENT_FAILED_EXTENSION_LOAD, extension_failed_load_handler) # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) # verify each installed extension is properly loaded if not FAILED_TO_LOAD or set(FAILED_TO_LOAD).issubset(set(EXTENSION_FAILURE_EXCLUSIONS)): print('Everything loaded successfully.') else: raise Exception('Exceptions failed to load: {}'.format(', '.join(FAILED_TO_LOAD)))
def main(): az_cli = AzCli(cli_name='az', commands_loader_cls=MainCommandsLoader, invocation_cls=AzCliCommandInvoker, parser_cls=AzCliCommandParser, help_cls=AzCliHelp) create_invoker_and_load_cmds_and_args(az_cli) help_files = get_all_help(az_cli) high_command_set = set() for help_file in help_files: if help_file.command: high_command_set.add(help_file.command.split()[0]) print('high_command_set:') print(high_command_set) # Load and check service_name.json with open('src/azure-cli/service_name.json') as f: service_names = json.load(f) print('Verifying src/azure-cli/service_name.json') service_name_map = {} for service_name in service_names: command = service_name['Command'] service = service_name['AzureServiceName'] if not command.startswith('az '): raise Exception('{} does not start with az!'.format(command)) if not service: raise Exception('AzureServiceName of {} is empty!'.format(command)) service_name_map[command[3:]] = service print('service_name_map:') print(service_name_map) # Check existence in service_name.json for high_command in high_command_set: if high_command not in service_name_map: raise Exception( 'No entry of {} in service_name.json. Please add one to the file.' .format(high_command))
def make_rst(self): INDENT = ' ' DOUBLEINDENT = INDENT * 2 az_cli = AzCli(cli_name='az', commands_loader_cls=MainCommandsLoader, invocation_cls=AzCliCommandInvoker, parser_cls=AzCliCommandParser, help_cls=AzCliHelp) with patch('getpass.getuser', return_value='your_system_user_login_name'): create_invoker_and_load_cmds_and_args(az_cli) help_files = get_all_help(az_cli) doc_source_map = _load_doc_source_map() for help_file in help_files: is_command = isinstance(help_file, CliCommandHelpFile) yield '.. cli{}:: {}'.format('command' if is_command else 'group', help_file.command if help_file.command else 'az') #it is top level group az if command is empty yield '' yield '{}:summary: {}'.format(INDENT, help_file.short_summary) yield '{}:description: {}'.format(INDENT, help_file.long_summary) if help_file.deprecate_info: yield '{}:deprecated: {}'.format(INDENT, help_file.deprecate_info._get_message(help_file.deprecate_info)) if not is_command: top_group_name = help_file.command.split()[0] if help_file.command else 'az' yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_group_name] if top_group_name in doc_source_map else '') else: top_command_name = help_file.command.split()[0] if help_file.command else '' if top_command_name in doc_source_map: yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_command_name]) yield '' if is_command and help_file.parameters: group_registry = ArgumentGroupRegistry( [p.group_name for p in help_file.parameters if p.group_name]) for arg in sorted(help_file.parameters, key=lambda p: group_registry.get_group_priority(p.group_name) + str(not p.required) + p.name): yield '{}.. cliarg:: {}'.format(INDENT, arg.name) yield '' yield '{}:required: {}'.format(DOUBLEINDENT, arg.required) if arg.deprecate_info: yield '{}:deprecated: {}'.format(DOUBLEINDENT, arg.deprecate_info._get_message(arg.deprecate_info)) short_summary = arg.short_summary or '' possible_values_index = short_summary.find(' Possible values include') short_summary = short_summary[0:possible_values_index if possible_values_index >= 0 else len(short_summary)] short_summary = short_summary.strip() yield '{}:summary: {}'.format(DOUBLEINDENT, short_summary) yield '{}:description: {}'.format(DOUBLEINDENT, arg.long_summary) if arg.choices: yield '{}:values: {}'.format(DOUBLEINDENT, ', '.join(sorted([str(x) for x in arg.choices]))) if arg.default and arg.default != argparse.SUPPRESS: try: if arg.default.startswith(USER_HOME): arg.default = arg.default.replace(USER_HOME, '~').replace('\\', '/') except Exception: pass try: arg.default = arg.default.replace("\\", "\\\\") except Exception: pass yield '{}:default: {}'.format(DOUBLEINDENT, arg.default) if arg.value_sources: yield '{}:source: {}'.format(DOUBLEINDENT, ', '.join(_get_populator_commands(arg))) yield '' yield '' if len(help_file.examples) > 0: for e in help_file.examples: yield '{}.. cliexample:: {}'.format(INDENT, e.short_summary) yield '' yield DOUBLEINDENT + e.command.replace("\\", "\\\\") yield ''
def run_linter(modules=None, rule_types=None, rules=None): require_azure_cli() from azure.cli.core import get_default_cli # pylint: disable=import-error from azure.cli.core.file_util import ( # pylint: disable=import-error get_all_help, create_invoker_and_load_cmds_and_args) heading('CLI Linter') # needed to remove helps from azdev azdev_helps = helps.copy() exclusions = {} selected_modules = get_path_table(include_only=modules) if not selected_modules: raise CLIError('No modules selected.') selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \ list(selected_modules['ext'].keys()) selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \ list(selected_modules['ext'].values()) if selected_mod_names: display('Modules: {}\n'.format(', '.join(selected_mod_names))) # collect all rule exclusions for path in selected_mod_paths: exclusion_path = os.path.join(path, 'linter_exclusions.yml') if os.path.isfile(exclusion_path): mod_exclusions = yaml.load(open(exclusion_path)) exclusions.update(mod_exclusions) start = time.time() display('Initializing linter with command table and help files...') az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) stop = time.time() logger.info('Commands and help loaded in %i sec', stop - start) command_loader = az_cli.invocation.commands_loader # format loaded help loaded_help = {data.command: data for data in loaded_help if data.command} # load yaml help help_file_entries = {} for entry_name, help_yaml in helps.items(): # ignore help entries from azdev itself, unless it also coincides # with a CLI or extension command name. if entry_name in azdev_helps and entry_name not in command_loader.command_table: continue help_entry = yaml.load(help_yaml) help_file_entries[entry_name] = help_entry # trim command table and help to just selected_modules command_loader, help_file_entries = filter_modules( command_loader, help_file_entries, modules=selected_mod_names) if not command_loader.command_table: raise CLIError('No commands selected to check.') # Instantiate and run Linter linter_manager = LinterManager(command_loader=command_loader, help_file_entries=help_file_entries, loaded_help=loaded_help, exclusions=exclusions, rule_inclusions=rules) subheading('Results') logger.info('Running linter: %i commands, %i help entries', len(command_loader.command_table), len(help_file_entries)) exit_code = linter_manager.run( run_params=not rule_types or 'params' in rule_types, run_commands=not rule_types or 'commands' in rule_types, run_command_groups=not rule_types or 'command_groups'in rule_types, run_help_files_entries=not rule_types or 'help_entries' in rule_types) sys.exit(exit_code)
def _get_help_files(self, az_cli): create_invoker_and_load_cmds_and_args(az_cli) return get_all_help(az_cli)
def run_linter(modules=None, rule_types=None, rules=None, ci_exclusions=None, git_source=None, git_target=None, git_repo=None, include_whl_extensions=False, min_severity=None, save_global_exclusion=False): require_azure_cli() from azure.cli.core import get_default_cli # pylint: disable=import-error from azure.cli.core.file_util import ( # pylint: disable=import-error get_all_help, create_invoker_and_load_cmds_and_args) heading('CLI Linter') # allow user to run only on CLI or extensions cli_only = modules == ['CLI'] ext_only = modules == ['EXT'] if cli_only or ext_only: modules = None # process severity option if min_severity: try: min_severity = LinterSeverity.get_linter_severity(min_severity) except ValueError: valid_choices = linter_severity_choices() raise CLIError( "Please specify a valid linter severity. It should be one of: {}" .format(", ".join(valid_choices))) # needed to remove helps from azdev azdev_helps = helps.copy() exclusions = {} selected_modules = get_path_table( include_only=modules, include_whl_extensions=include_whl_extensions) if cli_only: selected_modules['ext'] = {} if ext_only: selected_modules['mod'] = {} selected_modules['core'] = {} # used to upsert global exclusion update_global_exclusion = None if save_global_exclusion and (cli_only or ext_only): if cli_only: update_global_exclusion = 'CLI' if os.path.exists( os.path.join(get_cli_repo_path(), 'linter_exclusions.yml')): os.remove( os.path.join(get_cli_repo_path(), 'linter_exclusions.yml')) elif ext_only: update_global_exclusion = 'EXT' for ext_path in get_ext_repo_paths(): if os.path.exists( os.path.join(ext_path, 'linter_exclusions.yml')): os.remove(os.path.join(ext_path, 'linter_exclusions.yml')) # filter down to only modules that have changed based on git diff selected_modules = filter_by_git_diff(selected_modules, git_source, git_target, git_repo) if not any((selected_modules[x] for x in selected_modules)): raise CLIError('No modules selected.') selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \ list(selected_modules['ext'].keys()) selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \ list(selected_modules['ext'].values()) if selected_mod_names: display('Modules: {}\n'.format(', '.join(selected_mod_names))) # collect all rule exclusions for path in selected_mod_paths: exclusion_path = os.path.join(path, 'linter_exclusions.yml') if os.path.isfile(exclusion_path): mod_exclusions = yaml.safe_load(open(exclusion_path)) merge_exclusion(exclusions, mod_exclusions or {}) global_exclusion_paths = [ os.path.join(get_cli_repo_path(), 'linter_exclusions.yml') ] try: global_exclusion_paths.extend([ os.path.join(path, 'linter_exclusions.yml') for path in (get_ext_repo_paths() or []) ]) except CLIError: pass for path in global_exclusion_paths: if os.path.isfile(path): mod_exclusions = yaml.safe_load(open(path)) merge_exclusion(exclusions, mod_exclusions or {}) start = time.time() display('Initializing linter with command table and help files...') az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) stop = time.time() logger.info('Commands and help loaded in %i sec', stop - start) command_loader = az_cli.invocation.commands_loader # format loaded help loaded_help = {data.command: data for data in loaded_help if data.command} # load yaml help help_file_entries = {} for entry_name, help_yaml in helps.items(): # ignore help entries from azdev itself, unless it also coincides # with a CLI or extension command name. if entry_name in azdev_helps and entry_name not in command_loader.command_table: continue help_entry = yaml.safe_load(help_yaml) help_file_entries[entry_name] = help_entry # trim command table and help to just selected_modules command_loader, help_file_entries = filter_modules( command_loader, help_file_entries, modules=selected_mod_names, include_whl_extensions=include_whl_extensions) if not command_loader.command_table: raise CLIError('No commands selected to check.') # Instantiate and run Linter linter_manager = LinterManager( command_loader=command_loader, help_file_entries=help_file_entries, loaded_help=loaded_help, exclusions=exclusions, rule_inclusions=rules, use_ci_exclusions=ci_exclusions, min_severity=min_severity, update_global_exclusion=update_global_exclusion) subheading('Results') logger.info('Running linter: %i commands, %i help entries', len(command_loader.command_table), len(help_file_entries)) exit_code = linter_manager.run(run_params=not rule_types or 'params' in rule_types, run_commands=not rule_types or 'commands' in rule_types, run_command_groups=not rule_types or 'command_groups' in rule_types, run_help_files_entries=not rule_types or 'help_entries' in rule_types) sys.exit(exit_code)
def main(args): from azure.cli.core import get_default_cli from azure.cli.core.file_util import get_all_help, create_invoker_and_load_cmds_and_args print('Initializing linter with command table and help files...') # setup CLI to enable command loader az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) command_table = az_cli.invocation.commands_loader.command_table # format loaded help loaded_help = {data.command: data for data in loaded_help if data.command} # load yaml help help_file_entries = {} for entry_name, help_yaml in helps.items(): help_entry = yaml.load(help_yaml) help_file_entries[entry_name] = help_entry if not args.rule_types_to_run: args.rule_types_to_run = ['params', 'commands', 'command_groups', 'help_entries'] # find rule exclusions and pass to linter manager from ..utilities.path import get_command_modules_paths exclusions = {} command_modules_paths = get_command_modules_paths() for _, path in get_command_modules_paths(): exclusion_path = os.path.join(path, 'linter_exclusions.yml') if os.path.isfile(exclusion_path): mod_exclusions = yaml.load(open(exclusion_path)) exclusions.update(mod_exclusions) # only run linter on modules specified if args.modules: allowed_module_paths = tuple(path for mod, path in command_modules_paths if mod in args.modules) for command_name, command in list(command_table.items()): # brute force way to remove all traces from excluded modules loader_cls = command.loader.__class__ loader_file_path = inspect.getfile(loader_cls) if not loader_file_path.startswith(allowed_module_paths): del command_table[command_name] help_file_entries.pop(command_name, None) for group_name in _get_command_groups(command_name): help_file_entries.pop(group_name, None) # Instantiate and run Linter linter_manager = LinterManager(command_table=command_table, help_file_entries=help_file_entries, loaded_help=loaded_help, exclusions=exclusions) exit_code = linter_manager.run(run_params='params' in args.rule_types_to_run, run_commands='commands' in args.rule_types_to_run, run_command_groups='command_groups' in args.rule_types_to_run, run_help_files_entries='help_entries' in args.rule_types_to_run, ci=args.ci) sys.exit(exit_code)
def run_command_coverage(args): from azure.cli.core import get_default_cli from azure.cli.core.file_util import create_invoker_and_load_cmds_and_args from azure.cli.testsdk.base import ENV_COMMAND_COVERAGE, COVERAGE_FILE print(""" ******************************* * AZURE CLI 2.0 TEST COVERAGE * ******************************* This script is in development. Before using, ensure you have run your tests with the environment variable '{}' set to True! This script will not run the tests for you. Commands from test runs are aggregated in the '{}' file. """.format(ENV_COMMAND_COVERAGE, COVERAGE_FILE), file=sys.stderr) prefix = args.prefix report = args.report untested_params = args.untested_params projection = args.projection executed_commands = [] with open(COVERAGE_FILE, 'r') as f: executed_commands = f.readlines() coverage_data = _build_coverage_data(executed_commands) cli = get_default_cli() create_invoker_and_load_cmds_and_args(cli) command_table = cli.invocation.commands_loader.command_table def _lookup_dest(opt, command_args): for dest, arg in command_args.items(): if opt in arg.options_list: return dest all_commands = set(command_table.keys()) command_summary = {} commands_to_check = [x for x in all_commands if x.startswith(prefix)] for command in commands_to_check: # enter command into command summary if command not in command_summary: command_summary[command] = { 'command': command, 'tested': command in coverage_data, # if the command is tested 'live_only': None, # if the command is tested live only (less ideal) 'record_only': None, # if command is tested with @record_only (less ideal) 'param_score': 0.0, # num_tested_params / total_params 'tested_params': [], # list of tested params 'untested_params': [] # list of untested params } covered_params = coverage_data.get(command, []) command_arguments = command_table[command].arguments parameter_summary = {} for key, val in command_arguments.items(): # ignore generic update args if key in ['properties_to_add', 'properties_to_remove', 'properties_to_set']: continue # ignore suppressed args if val.type.settings.get('help', '') == '==SUPPRESS==': continue parameter_summary[key] = False for cp in covered_params: dest = _lookup_dest(cp, command_arguments) parameter_summary[dest] = True for k, v in parameter_summary.items(): if not v: command_summary[command]['untested_params'].append(k) else: command_summary[command]['tested_params'].append(k) total_param_count = len(parameter_summary) tested_param_count = total_param_count - len(command_summary[command]['untested_params']) command_summary[command]['param_score'] = tested_param_count * 1.0 / total_param_count if \ total_param_count else 1.0 # Filter results if prefix: command_summary = {k: v for k, v in command_summary.items() if k.startswith(prefix)} if untested_params: command_summary = {k: v for k, v in command_summary.items() if untested_params in v['untested_params']} if not report: final_values = [] if projection: for entry in command_summary.values(): final_values.append({k: v for k, v in entry.items() if k in projection}) if not final_values: final_values = list(command_summary.values()) print(json.dumps(final_values, indent=4, sort_keys=True)) else: report_summary = _build_command_report(command_summary) spacing_format = '{0:<20s} {1:<40s} {2:<10s} {3:<10s}' print(spacing_format.format('MODULE', 'GROUP', 'COMMAND', 'PARAM')) for mod, mod_data in report_summary.items(): for group, group_data in mod_data.items(): print(spacing_format.format( mod, group, group_data['command_score'], group_data['param_score']))