def publish_extensions(extensions, storage_subscription, storage_account, storage_container, dist_dir='dist', update_index=False, yes=False): heading('Publish Extensions') require_azure_cli() # rebuild the extensions subheading('Building WHLs') try: shutil.rmtree(dist_dir) except Exception as ex: # pylint: disable=broad-except logger.debug("Unable to clear folder '%s'. Error: %s", dist_dir, ex) build_extensions(extensions, dist_dir=dist_dir) whl_files = find_files(dist_dir, '*.whl') uploaded_urls = [] subheading('Uploading WHLs') for whl_path in whl_files: whl_file = os.path.split(whl_path)[-1] # check if extension already exists unless user opted not to if not yes: command = 'az storage blob exists --subscription {} --account-name {} -c {} -n {}'.format( storage_subscription, storage_account, storage_container, whl_file) exists = json.loads(cmd(command).result)['exists'] if exists: if not prompt_y_n( "{} already exists. You may need to bump the extension version. Replace?" .format(whl_file), default='n'): logger.warning("Skipping '%s'...", whl_file) continue # upload the WHL file command = 'az storage blob upload --subscription {} --account-name {} -c {} -n {} -f {}'.format( storage_subscription, storage_account, storage_container, whl_file, os.path.abspath(whl_path)) cmd(command, "Uploading '{}'...".format(whl_file)) command = 'az storage blob url --subscription {} --account-name {} -c {} -n {} -otsv'.format( storage_subscription, storage_account, storage_container, whl_file) url = cmd(command).result logger.info(url) uploaded_urls.append(url) if update_index: subheading('Updating Index') update_extension_index(uploaded_urls) subheading('Published') display(uploaded_urls) if not update_index: logger.warning( 'You still need to update the index for your changes with `az extension update-index`.' )
def check_style(modules=None, pylint=False, pep8=False): heading('Style Check') selected_modules = get_path_table(include_only=modules) pep8_result = None pylint_result = None if pylint: try: require_azure_cli() except CLIError: raise CLIError( 'usage error: --pylint requires Azure CLI to be installed.') if not selected_modules: raise CLIError('No modules selected.') mod_names = list(selected_modules['mod'].keys()) + list( selected_modules['core'].keys()) ext_names = list(selected_modules['ext'].keys()) if mod_names: display('Modules: {}\n'.format(', '.join(mod_names))) if ext_names: display('Extensions: {}\n'.format(', '.join(ext_names))) # if neither flag provided, same as if both were provided if not any([pylint, pep8]): pep8 = True pylint = True exit_code_sum = 0 if pep8: pep8_result = _run_pep8(selected_modules) exit_code_sum += pep8_result.exit_code if pylint: pylint_result = _run_pylint(selected_modules) exit_code_sum += pylint_result.exit_code display('') subheading('Results') # print success messages first if pep8_result and not pep8_result.error: display('Flake8: PASSED') if pylint_result and not pylint_result.error: display('Pylint: PASSED') display('') # print error messages last if pep8_result and pep8_result.error: logger.error(pep8_result.error.output.decode('utf-8')) logger.error('Flake8: FAILED\n') if pylint_result and pylint_result.error: logger.error(pylint_result.error.output.decode('utf-8')) logger.error('Pylint: FAILED\n') sys.exit(exit_code_sum)
def delete_groups(prefixes=None, older_than=6, product='azurecli', cause='automation', yes=False): from datetime import datetime, timedelta require_azure_cli() groups = json.loads(run_cmd('az group list -ojson').result) groups_to_delete = [] def _filter_by_tags(): for group in groups: group = Data(**group) if not group.tags: # pylint: disable=no-member continue tags = Data(**group.tags) # pylint: disable=no-member try: date_tag = datetime.strptime(tags.date, '%Y-%m-%dT%H:%M:%SZ') curr_time = datetime.utcnow() if (tags.product == product and tags.cause == cause and (curr_time - date_tag <= timedelta(hours=older_than + 1))): groups_to_delete.append(group.name) except AttributeError: continue def _filter_by_prefix(): for group in groups: group = Data(**group) for prefix in prefixes: if group.name.startswith(prefix): groups_to_delete.append(group.name) def _delete(): for group in groups_to_delete: run_cmd('az group delete -g {} -y --no-wait'.format(group), message=True) if prefixes: logger.info('Filter by prefix') _filter_by_prefix() else: logger.info('Filter by tags') _filter_by_tags() if not groups_to_delete: raise CLIError('No groups meet the criteria to delete.') if yes: _delete() else: subheading('Groups to Delete') for group in groups_to_delete: display('\t{}'.format(group)) if prompt_y_n('Delete {} resource groups?'.format(len(groups_to_delete)), 'y'): _delete() else: raise CLIError('Command cancelled.')
def verify_versions(): import tempfile import shutil require_azure_cli() heading('Verify CLI Versions') path_table = get_path_table() modules = list(path_table['core'].items()) modules = [x for x in modules if x[0] not in EXCLUDED_MODULES] if not modules: raise CLIError('No modules selected to test.') display('MODULES: {}'.format(', '.join([x[0] for x in modules]))) results = {} original_cwd = os.getcwd() temp_dir = tempfile.mkdtemp() for mod, mod_path in modules: if not mod.startswith(COMMAND_MODULE_PREFIX) and mod != 'azure-cli': mod = '{}{}'.format(COMMAND_MODULE_PREFIX, mod) results[mod] = {} results.update( _compare_module_against_pypi(results, temp_dir, mod, mod_path)) shutil.rmtree(temp_dir) os.chdir(original_cwd) logger.info('Module'.ljust(40) + 'Local Version'.rjust(20) + 'Public Version'.rjust(20)) # pylint: disable=logging-not-lazy for mod, data in results.items(): logger.info( mod.ljust(40) + data['local_version'].rjust(20) + data['public_version'].rjust(20)) bump_mods = {k: v for k, v in results.items() if v['status'] == 'BUMP'} subheading('RESULTS') if bump_mods: logger.error( 'The following modules need their versions bumped. ' 'Scroll up for details: %s', ', '.join(bump_mods.keys())) logger.warning( '\nNote that before changing versions, you should consider ' 'running `git clean` to remove untracked files from your repo. ' 'Files that were once tracked but removed from the source may ' 'still be on your machine, resuling in false positives.') sys.exit(1) else: display('OK!')
def update_setup_py(pin=False): require_azure_cli() heading('Update azure-cli setup.py') path_table = get_path_table() azure_cli_path = path_table['core']['azure-cli'] azure_cli_setup_path = find_files(azure_cli_path, SETUP_PY_NAME)[0] modules = list(path_table['core'].items()) + list(path_table['mod'].items()) modules = [x for x in modules if x[0] not in EXCLUDED_MODULES] results = {mod[0]: {} for mod in modules} results = _get_module_versions(results, modules) _update_setup_py(results, azure_cli_setup_path, pin) display('OK!')
def check_load_time(runs=3): require_azure_cli() heading('Module Load Performance') regex = r"[^']*'([^']*)'[\D]*([\d\.]*)" results = {TOTAL: []} # Time the module loading X times for i in range(0, runs + 1): lines = cmd('az -h --debug', show_stderr=True).result if i == 0: # Ignore the first run since it can be longer due to *.pyc file compilation continue try: lines = lines.decode().splitlines() except AttributeError: lines = lines.splitlines() total_time = 0 for line in lines: if line.startswith('DEBUG: Loaded module'): matches = re.match(regex, line) mod = matches.group(1) val = float(matches.group(2)) * 1000 total_time = total_time + val if mod in results: results[mod].append(val) else: results[mod] = [val] results[TOTAL].append(total_time) passed_mods = {} failed_mods = {} mods = sorted(results.keys()) bubble_found = False for mod in mods: val = results[mod] mean_val = mean(val) stdev_val = pstdev(val) threshold = THRESHOLDS.get(mod) or DEFAULT_THRESHOLD statistics = { 'average': mean_val, 'stdev': stdev_val, 'threshold': threshold, 'values': val } if mean_val > threshold: if not bubble_found and mean_val < 30: # This temporary measure allows one floating performance # failure up to 30 ms. See issue #6224 and #6218. bubble_found = True passed_mods[mod] = statistics else: failed_mods[mod] = statistics else: passed_mods[mod] = statistics subheading('Results') if failed_mods: display('== PASSED MODULES ==') display_table(passed_mods) display('\nFAILED MODULES') display_table(failed_mods) raise CLIError(""" FAILED: Some modules failed. If values are close to the threshold, rerun. If values are large, check that you do not have top-level imports like azure.mgmt or msrestazure in any modified files. """) display('== PASSED MODULES ==') display_table(passed_mods) display('\nPASSED: Average load time all modules: {} ms'.format( int(passed_mods[TOTAL]['average'])))
def check_load_time(runs=3): require_azure_cli() heading('Module Load Performance') regex = r"[^']*'(?P<mod>[^']*)'[\D]*(?P<val>[\d\.]*)" results = {TOTAL: []} # Time the module loading X times for i in range(0, runs + 1): lines = cmd('az -h --debug', show_stderr=True).result if i == 0: # Ignore the first run since it can be longer due to *.pyc file compilation continue try: lines = lines.decode().splitlines() except AttributeError: lines = lines.splitlines() total_time = 0 for line in lines: if line.startswith('DEBUG: Loaded module'): matches = re.match(regex, line) mod = matches.group('mod') val = float(matches.group('val')) * 1000 total_time = total_time + val if mod in results: results[mod].append(val) else: results[mod] = [val] results[TOTAL].append(total_time) passed_mods = {} failed_mods = {} def _claim_higher_threshold(val): avail_thresholds = {k: v for k, v in THRESHOLDS.items() if v} new_threshold = None for threshold in sorted(avail_thresholds): if val < threshold: THRESHOLDS[threshold] = THRESHOLDS[threshold] - 1 new_threshold = threshold break return new_threshold mods = sorted(results.keys()) for mod in mods: val = results[mod] mean_val = mean(val) stdev_val = pstdev(val) threshold = TOTAL_THRESHOLD if mod == TOTAL else DEFAULT_THRESHOLD statistics = { 'average': mean_val, 'stdev': stdev_val, 'threshold': threshold, 'values': val } if mean_val > threshold: # claim a threshold exception if available new_threshold = _claim_higher_threshold(mean_val) if new_threshold: statistics['threshold'] = new_threshold passed_mods[mod] = statistics else: failed_mods[mod] = statistics else: passed_mods[mod] = statistics subheading('Results') if failed_mods: display('== PASSED MODULES ==') display_table(passed_mods) display('\nFAILED MODULES') display_table(failed_mods) raise CLIError(""" FAILED: Some modules failed. If values are close to the threshold, rerun. If values are large, check that you do not have top-level imports like azure.mgmt or msrestazure in any modified files. """) display('== PASSED MODULES ==') display_table(passed_mods) display('\nPASSED: Average load time all modules: {} ms'.format( int(passed_mods[TOTAL]['average'])))
def check_style(modules=None, pylint=False, pep8=False, git_source=None, git_target=None, git_repo=None): heading('Style Check') # allow user to run only on CLI or extensions cli_only = modules == ['CLI'] ext_only = modules == ['EXT'] if cli_only or ext_only: modules = None selected_modules = get_path_table(include_only=modules) # remove these two non-modules selected_modules['core'].pop('azure-cli-nspkg', None) selected_modules['core'].pop('azure-cli-command_modules-nspkg', None) pep8_result = None pylint_result = None if pylint: try: require_azure_cli() except CLIError: raise CLIError('usage error: --pylint requires Azure CLI to be installed.') if cli_only: ext_names = None selected_modules['ext'] = {} if ext_only: mod_names = None selected_modules['mod'] = {} selected_modules['core'] = {} # filter down to only modules that have changed based on git diff selected_modules = filter_by_git_diff(selected_modules, git_source, git_target, git_repo) if not any((selected_modules[x] for x in selected_modules)): raise CLIError('No modules selected.') mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) ext_names = list(selected_modules['ext'].keys()) if mod_names: display('Modules: {}\n'.format(', '.join(mod_names))) if ext_names: display('Extensions: {}\n'.format(', '.join(ext_names))) # if neither flag provided, same as if both were provided if not any([pylint, pep8]): pep8 = True pylint = True exit_code_sum = 0 if pep8: pep8_result = _run_pep8(selected_modules) exit_code_sum += pep8_result.exit_code if pylint: pylint_result = _run_pylint(selected_modules) exit_code_sum += pylint_result.exit_code display('') subheading('Results') # print success messages first if pep8_result and not pep8_result.error: display('Flake8: PASSED') if pylint_result and not pylint_result.error: display('Pylint: PASSED') display('') # print error messages last if pep8_result and pep8_result.error: logger.error(pep8_result.error.output.decode('utf-8')) logger.error('Flake8: FAILED\n') if pylint_result and pylint_result.error: logger.error(pylint_result.error.output.decode('utf-8')) logger.error('Pylint: FAILED\n') sys.exit(exit_code_sum)
def run_linter(modules=None, rule_types=None, rules=None): require_azure_cli() from azure.cli.core import get_default_cli # pylint: disable=import-error from azure.cli.core.file_util import ( # pylint: disable=import-error get_all_help, create_invoker_and_load_cmds_and_args) heading('CLI Linter') # needed to remove helps from azdev azdev_helps = helps.copy() exclusions = {} selected_modules = get_path_table(include_only=modules) if not selected_modules: raise CLIError('No modules selected.') selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \ list(selected_modules['ext'].keys()) selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \ list(selected_modules['ext'].values()) if selected_mod_names: display('Modules: {}\n'.format(', '.join(selected_mod_names))) # collect all rule exclusions for path in selected_mod_paths: exclusion_path = os.path.join(path, 'linter_exclusions.yml') if os.path.isfile(exclusion_path): mod_exclusions = yaml.load(open(exclusion_path)) exclusions.update(mod_exclusions) start = time.time() display('Initializing linter with command table and help files...') az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) stop = time.time() logger.info('Commands and help loaded in %i sec', stop - start) command_loader = az_cli.invocation.commands_loader # format loaded help loaded_help = {data.command: data for data in loaded_help if data.command} # load yaml help help_file_entries = {} for entry_name, help_yaml in helps.items(): # ignore help entries from azdev itself, unless it also coincides # with a CLI or extension command name. if entry_name in azdev_helps and entry_name not in command_loader.command_table: continue help_entry = yaml.load(help_yaml) help_file_entries[entry_name] = help_entry # trim command table and help to just selected_modules command_loader, help_file_entries = filter_modules( command_loader, help_file_entries, modules=selected_mod_names) if not command_loader.command_table: raise CLIError('No commands selected to check.') # Instantiate and run Linter linter_manager = LinterManager(command_loader=command_loader, help_file_entries=help_file_entries, loaded_help=loaded_help, exclusions=exclusions, rule_inclusions=rules) subheading('Results') logger.info('Running linter: %i commands, %i help entries', len(command_loader.command_table), len(help_file_entries)) exit_code = linter_manager.run( run_params=not rule_types or 'params' in rule_types, run_commands=not rule_types or 'commands' in rule_types, run_command_groups=not rule_types or 'command_groups'in rule_types, run_help_files_entries=not rule_types or 'help_entries' in rule_types) sys.exit(exit_code)
def publish_extensions(extensions, storage_account, storage_account_key, storage_container, dist_dir='dist', update_index=False, yes=False): from azure.multiapi.storage.v2018_11_09.blob import BlockBlobService heading('Publish Extensions') require_azure_cli() # rebuild the extensions subheading('Building WHLs') try: shutil.rmtree(dist_dir) except Exception as ex: # pylint: disable=broad-except logger.debug("Unable to clear folder '%s'. Error: %s", dist_dir, ex) build_extensions(extensions, dist_dir=dist_dir) whl_files = find_files(dist_dir, '*.whl') uploaded_urls = [] subheading('Uploading WHLs') for whl_path in whl_files: whl_file = os.path.split(whl_path)[-1] client = BlockBlobService(account_name=storage_account, account_key=storage_account_key) exists = client.exists(container_name=storage_container, blob_name=whl_file) # check if extension already exists unless user opted not to if not yes: if exists: if not prompt_y_n( "{} already exists. You may need to bump the extension version. Replace?" .format(whl_file), default='n'): logger.warning("Skipping '%s'...", whl_file) continue # upload the WHL file client.create_blob_from_path(container_name=storage_container, blob_name=whl_file, file_path=os.path.abspath(whl_path)) url = client.make_blob_url(container_name=storage_container, blob_name=whl_file) logger.info(url) uploaded_urls.append(url) if update_index: subheading('Updating Index') update_extension_index(uploaded_urls) subheading('Published WHLs') for url in uploaded_urls: display(url) if not update_index: logger.warning('You still need to update the index for your changes!') logger.warning(' az extension update-index <URL>')
def verify_versions(modules=None, update=False, pin=False): import tempfile import shutil require_azure_cli() heading('Verify CLI Module Versions') usage_err = CLIError('usage error: <MODULES> | --update [--pin]') if modules and (update or pin): raise usage_err if not modules and pin and not update: raise usage_err if modules: update = None pin = None path_table = get_path_table(include_only=modules) modules = list(path_table['core'].items()) + list(path_table['mod'].items()) modules = [x for x in modules if x[0] not in EXCLUDED_MODULES] if not modules: raise CLIError('No modules selected to test.') display('MODULES: {}'.format(', '.join([x[0] for x in modules]))) results = {mod[0]: {} for mod in modules} original_cwd = os.getcwd() temp_dir = tempfile.mkdtemp() for mod, mod_path in modules: if not mod.startswith(COMMAND_MODULE_PREFIX) and mod != 'azure-cli': mod = '{}{}'.format(COMMAND_MODULE_PREFIX, mod) results.update(_compare_module_against_pypi(results, temp_dir, mod, mod_path)) shutil.rmtree(temp_dir) os.chdir(original_cwd) results = _check_setup_py(results, update, pin) logger.info('Module'.ljust(40) + 'Local Version'.rjust(20) + 'Public Version'.rjust(20)) # pylint: disable=logging-not-lazy for mod, data in results.items(): logger.info(mod.ljust(40) + data['local_version'].rjust(20) + data['public_version'].rjust(20)) bump_mods = {k: v for k, v in results.items() if v['status'] == 'BUMP'} mismatch_mods = {k: v for k, v in results.items() if v['status'] == 'MISMATCH'} subheading('RESULTS') if bump_mods: logger.error('The following modules need their versions bumped. ' 'Scroll up for details: %s', ', '.join(bump_mods.keys())) logger.warning('\nNote that before changing versions, you should consider ' 'running `git clean` to remove untracked files from your repo. ' 'Files that were once tracked but removed from the source may ' 'still be on your machine, resuling in false positives.') sys.exit(1) elif mismatch_mods and not update: logger.error('The following modules have a mismatch between the module version ' 'and the version in azure-cli\'s setup.py file. ' 'Scroll up for details: %s', ', '.join(mismatch_mods.keys())) sys.exit(1) else: display('OK!')
def run_linter(modules=None, rule_types=None, rules=None, ci_exclusions=None, git_source=None, git_target=None, git_repo=None, include_whl_extensions=False, min_severity=None, save_global_exclusion=False): require_azure_cli() from azure.cli.core import get_default_cli # pylint: disable=import-error from azure.cli.core.file_util import ( # pylint: disable=import-error get_all_help, create_invoker_and_load_cmds_and_args) heading('CLI Linter') # allow user to run only on CLI or extensions cli_only = modules == ['CLI'] ext_only = modules == ['EXT'] if cli_only or ext_only: modules = None # process severity option if min_severity: try: min_severity = LinterSeverity.get_linter_severity(min_severity) except ValueError: valid_choices = linter_severity_choices() raise CLIError( "Please specify a valid linter severity. It should be one of: {}" .format(", ".join(valid_choices))) # needed to remove helps from azdev azdev_helps = helps.copy() exclusions = {} selected_modules = get_path_table( include_only=modules, include_whl_extensions=include_whl_extensions) if cli_only: selected_modules['ext'] = {} if ext_only: selected_modules['mod'] = {} selected_modules['core'] = {} # used to upsert global exclusion update_global_exclusion = None if save_global_exclusion and (cli_only or ext_only): if cli_only: update_global_exclusion = 'CLI' if os.path.exists( os.path.join(get_cli_repo_path(), 'linter_exclusions.yml')): os.remove( os.path.join(get_cli_repo_path(), 'linter_exclusions.yml')) elif ext_only: update_global_exclusion = 'EXT' for ext_path in get_ext_repo_paths(): if os.path.exists( os.path.join(ext_path, 'linter_exclusions.yml')): os.remove(os.path.join(ext_path, 'linter_exclusions.yml')) # filter down to only modules that have changed based on git diff selected_modules = filter_by_git_diff(selected_modules, git_source, git_target, git_repo) if not any((selected_modules[x] for x in selected_modules)): raise CLIError('No modules selected.') selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \ list(selected_modules['ext'].keys()) selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \ list(selected_modules['ext'].values()) if selected_mod_names: display('Modules: {}\n'.format(', '.join(selected_mod_names))) # collect all rule exclusions for path in selected_mod_paths: exclusion_path = os.path.join(path, 'linter_exclusions.yml') if os.path.isfile(exclusion_path): mod_exclusions = yaml.safe_load(open(exclusion_path)) merge_exclusion(exclusions, mod_exclusions or {}) global_exclusion_paths = [ os.path.join(get_cli_repo_path(), 'linter_exclusions.yml') ] try: global_exclusion_paths.extend([ os.path.join(path, 'linter_exclusions.yml') for path in (get_ext_repo_paths() or []) ]) except CLIError: pass for path in global_exclusion_paths: if os.path.isfile(path): mod_exclusions = yaml.safe_load(open(path)) merge_exclusion(exclusions, mod_exclusions or {}) start = time.time() display('Initializing linter with command table and help files...') az_cli = get_default_cli() # load commands, args, and help create_invoker_and_load_cmds_and_args(az_cli) loaded_help = get_all_help(az_cli) stop = time.time() logger.info('Commands and help loaded in %i sec', stop - start) command_loader = az_cli.invocation.commands_loader # format loaded help loaded_help = {data.command: data for data in loaded_help if data.command} # load yaml help help_file_entries = {} for entry_name, help_yaml in helps.items(): # ignore help entries from azdev itself, unless it also coincides # with a CLI or extension command name. if entry_name in azdev_helps and entry_name not in command_loader.command_table: continue help_entry = yaml.safe_load(help_yaml) help_file_entries[entry_name] = help_entry # trim command table and help to just selected_modules command_loader, help_file_entries = filter_modules( command_loader, help_file_entries, modules=selected_mod_names, include_whl_extensions=include_whl_extensions) if not command_loader.command_table: raise CLIError('No commands selected to check.') # Instantiate and run Linter linter_manager = LinterManager( command_loader=command_loader, help_file_entries=help_file_entries, loaded_help=loaded_help, exclusions=exclusions, rule_inclusions=rules, use_ci_exclusions=ci_exclusions, min_severity=min_severity, update_global_exclusion=update_global_exclusion) subheading('Results') logger.info('Running linter: %i commands, %i help entries', len(command_loader.command_table), len(help_file_entries)) exit_code = linter_manager.run(run_params=not rule_types or 'params' in rule_types, run_commands=not rule_types or 'commands' in rule_types, run_command_groups=not rule_types or 'command_groups' in rule_types, run_help_files_entries=not rule_types or 'help_entries' in rule_types) sys.exit(exit_code)