def remove_extension(extensions): ext_paths = get_ext_repo_paths() installed_paths = find_files(ext_paths, '*.*-info') paths_to_remove = [] names_to_remove = [] if extensions == ['*']: paths_to_remove = [os.path.dirname(path) for path in installed_paths] names_to_remove = [ os.path.basename(os.path.dirname(path)) for path in installed_paths ] else: for path in installed_paths: folder = os.path.dirname(path) long_name = os.path.basename(folder) if long_name in extensions: paths_to_remove.append(folder) names_to_remove.append(long_name) extensions.remove(long_name) # raise error if any extension not installed if extensions: raise CLIError('extension(s) not installed: {}'.format( ' '.join(extensions))) # removes any links that may have been added to site-packages. for ext in names_to_remove: pip_cmd('uninstall {} -y'.format(ext)) for path in paths_to_remove: for d in os.listdir(path): # delete the egg-info and dist-info folders to make the extension invisible to the CLI and azdev if d.endswith('egg-info') or d.endswith('dist-info'): path_to_remove = os.path.join(path, d) display("Removing '{}'...".format(path_to_remove)) shutil.rmtree(path_to_remove)
def display_table(data): display('{:<20} {:>12} {:>12} {:>12} {:>25}'.format( 'Module', 'Average', 'Threshold', 'Stdev', 'Values')) for key, val in data.items(): display('{:<20} {:>12.0f} {:>12.0f} {:>12.0f} {:>25}'.format( key, val['average'], val['threshold'], val['stdev'], str(val['values'])))
def cmd(command, message=False, show_stderr=True, **kwargs): """ Run an arbitrary command. :param command: The entire command line to run. :param message: A custom message to display, or True (bool) to use a default. :param show_stderr: On error, display the contents of STDERR. :param kwargs: Any kwargs supported by subprocess.Popen :returns: CommandResultItem object. """ from azdev.utilities import IS_WINDOWS, display # use default message if custom not provided if message is True: message = 'Running: {}\n'.format(command) if message: display(message) try: output = subprocess.check_output( command.split(), stderr=subprocess.STDOUT if show_stderr else None, shell=IS_WINDOWS, **kwargs).decode('utf-8').strip() return CommandResultItem(output, exit_code=0, error=None) except subprocess.CalledProcessError as err: return CommandResultItem(err.output, exit_code=err.returncode, error=err)
def _get_profile(profile): import colorama colorama.init(autoreset=True) try: fore_red = colorama.Fore.RED if not IS_WINDOWS else '' fore_reset = colorama.Fore.RESET if not IS_WINDOWS else '' current_profile = raw_cmd('az cloud show --query profile -otsv', show_stderr=False).result if not profile or current_profile == profile: profile = current_profile display( 'The tests are set to run against current profile {}.'.format( fore_red + current_profile + fore_reset)) elif current_profile != profile: display( 'The tests are set to run against profile {} but the current az cloud profile is {}.' .format(fore_red + profile + fore_reset, fore_red + current_profile + fore_reset)) result = raw_cmd( 'az cloud update --profile {}'.format(profile), 'SWITCHING TO PROFILE {}.'.format(fore_red + profile + fore_reset)) if result.exit_code != 0: raise CLIError(result.error.output) return current_profile except CalledProcessError: raise CLIError('Failed to retrieve current az profile')
def install_extensions(venv_path, extensions): activate_path = os.path.join( venv_path, 'Scripts', 'activate') if const.IS_WINDOWS else 'source ' + os.path.join( venv_path, const.UN_BIN, const.UN_ACTIVATE) delimiter = ' && ' if const.IS_WINDOWS else '; ' executable = None if const.IS_WINDOWS else '/bin/bash' all_ext = azdev.operations.extensions.list_extensions() if extensions == ['*']: display("\nInstalling all extensions") for i in all_ext: shell_cmd(activate_path + delimiter + const.PIP_E_CMD + i['path'], executable=executable) extensions = False else: display("\nInstalling the following extensions: " + str(extensions)) extensions = set(extensions) k = 0 while k < len(all_ext) and extensions: if all_ext[k]['name'] in extensions: shell_cmd(activate_path + delimiter + const.PIP_E_CMD + all_ext[k]['path'], executable=executable) extensions.remove(all_ext[k]['name']) k += 1 if extensions: raise CLIError( "The following extensions were not found. Ensure you have added " "the repo using `--repo/-r PATH`.\n {}".format( '\n '.join(extensions)))
def check_license_headers(): heading('Verify License Headers') cli_path = get_cli_repo_path() env_path = os.path.join(cli_path, 'env') files_without_header = [] for current_dir, _, files in os.walk(cli_path): if current_dir.startswith(env_path): continue file_itr = (os.path.join(current_dir, p) for p in files if p.endswith('.py') and p != 'azure_bdist_wheel.py') for python_file in file_itr: with open(python_file, 'r') as f: file_text = f.read() if file_text and LICENSE_HEADER not in file_text: files_without_header.append( os.path.join(current_dir, python_file)) subheading('Results') if files_without_header: raise CLIError( "{}\nError: {} files don't have the required license headers.". format('\n'.join(files_without_header), len(files_without_header))) display('License headers verified OK.')
def publish_extensions(extensions, storage_subscription, storage_account, storage_container, dist_dir='dist', update_index=False, yes=False): heading('Publish Extensions') require_azure_cli() # rebuild the extensions subheading('Building WHLs') try: shutil.rmtree(dist_dir) except Exception as ex: # pylint: disable=broad-except logger.debug("Unable to clear folder '%s'. Error: %s", dist_dir, ex) build_extensions(extensions, dist_dir=dist_dir) whl_files = find_files(dist_dir, '*.whl') uploaded_urls = [] subheading('Uploading WHLs') for whl_path in whl_files: whl_file = os.path.split(whl_path)[-1] # check if extension already exists unless user opted not to if not yes: command = 'az storage blob exists --subscription {} --account-name {} -c {} -n {}'.format( storage_subscription, storage_account, storage_container, whl_file) exists = json.loads(cmd(command).result)['exists'] if exists: if not prompt_y_n( "{} already exists. You may need to bump the extension version. Replace?" .format(whl_file), default='n'): logger.warning("Skipping '%s'...", whl_file) continue # upload the WHL file command = 'az storage blob upload --subscription {} --account-name {} -c {} -n {} -f {}'.format( storage_subscription, storage_account, storage_container, whl_file, os.path.abspath(whl_path)) cmd(command, "Uploading '{}'...".format(whl_file)) command = 'az storage blob url --subscription {} --account-name {} -c {} -n {} -otsv'.format( storage_subscription, storage_account, storage_container, whl_file) url = cmd(command).result logger.info(url) uploaded_urls.append(url) if update_index: subheading('Updating Index') update_extension_index(uploaded_urls) subheading('Published') display(uploaded_urls) if not update_index: logger.warning( 'You still need to update the index for your changes with `az extension update-index`.' )
def _generate_ref_docs_for_all_profiles(output_type, base_output_dir): original_profile = None profile = "" try: # store original profile and get all profiles. original_profile = _get_current_profile() profiles = _get_profiles() _logger.info("Original Profile: %s", original_profile) for profile in profiles: # set profile and call sphinx build cmd profile_output_dir = os.path.join(base_output_dir, profile) _set_profile(profile) _call_sphinx_build(output_type, profile_output_dir) display("\nFinished generating files for profile {} in dir {}\n". format(output_type, profile_output_dir)) # always set the profile back to the original profile after generating all docs. _set_profile(original_profile) except (CLIError, KeyboardInterrupt, SystemExit) as e: _logger.error( "Error when attempting to generate docs for profile %s.\n\t%s", profile, e) if original_profile: _logger.error( "Will try to set the CLI's profile back to the original value: '%s'", original_profile) _set_profile(original_profile) # still re-raise the error. raise e
def check_document_map(): heading('Verify Document Map') cli_repo = get_cli_repo_path() map_path = os.path.join(cli_repo, DOC_SOURCE_MAP_PATH) help_files_in_map = _get_help_files_in_map(map_path) help_files_not_found = _map_help_files_not_found(cli_repo, help_files_in_map) help_files_to_add_to_map = _help_files_not_in_map(cli_repo, help_files_in_map) subheading('Results') if help_files_not_found or help_files_to_add_to_map: error_lines = [] error_lines.append('Errors whilst verifying {}!'.format(DOC_MAP_NAME)) if help_files_not_found: error_lines.append( 'The following files are in {} but do not exist:'.format( DOC_MAP_NAME)) error_lines += help_files_not_found if help_files_to_add_to_map: error_lines.append( 'The following files should be added to {}:'.format( DOC_MAP_NAME)) error_lines += help_files_to_add_to_map error_msg = '\n'.join(error_lines) raise CLIError(error_msg) display('Verified {} OK.'.format(DOC_MAP_NAME))
def _call_sphinx_build(builder_name, output_dir, for_extensions_alone=False, call_env=None, msg=""): conf_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'refdoc') if for_extensions_alone: source_dir = os.path.abspath(os.path.join(conf_dir, 'extension_docs')) else: source_dir = os.path.abspath(os.path.join(conf_dir, 'cli_docs')) try: opts = ['-E', '-b', builder_name, '-c', conf_dir] args = [source_dir, output_dir] if for_extensions_alone: # apparently the configuration in extensions and core CLI differed in this way. This is only cosmetic # set smartquotes to false. Due to a bug, one has to use "0" instead "False" opts.extend(["-D", "smartquotes=0"]) sphinx_cmd = ['sphinx-build'] + opts + args display("sphinx cmd: {}".format(" ".join(sphinx_cmd))) display(msg) # call sphinx-build check_call(sphinx_cmd, stdout=sys.stdout, stderr=sys.stderr, env=call_env) except CalledProcessError: raise CLIError("Doc generation failed.")
def _run(test_paths, pytest_args): if os.name == 'posix': arguments = [ '-x', '-v', '--boxed', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path ] else: arguments = [ '-x', '-v', '-p no:warnings', '--log-level=WARN', '--junit-xml', log_path ] if no_exit_first: arguments.remove('-x') if mark: arguments.append('-m "{}"'.format(mark)) if parallel: arguments += ['-n', 'auto'] if last_failed: arguments.append('--lf') if pytest_args: arguments += pytest_args tests_params = [(i, clean, arguments) for i in test_paths] test_fail = False with multiprocessing.Pool(multiprocessing.cpu_count()) as the_pool: try: the_pool.map(_run_test, tests_params) except subprocess.CalledProcessError: display("TEST FAILED! : " + str(subprocess.CalledProcessError.stdout)) test_fail = True return test_fail
def _add_to_codeowners(repo_path, prefix, name, github_alias): # add the user Github alias to the CODEOWNERS file for new packages if not github_alias: display( '\nWhat is the Github alias of the person responsible for maintaining this package?' ) while not github_alias: github_alias = prompt('Alias: ') # accept a raw alias or @alias github_alias = '@{}'.format( github_alias) if not github_alias.startswith('@') else github_alias try: codeowners = find_files(repo_path, 'CODEOWNERS')[0] except IndexError: raise CLIError('unexpected error: unable to find CODEOWNERS file.') if prefix == EXTENSION_PREFIX: new_line = '/src/{}{}/ {}'.format(prefix, name, github_alias) else: # ensure Linux-style separators when run on Windows new_line = '/{} {}'.format( os.path.join('', _MODULE_ROOT_PATH, name, ''), github_alias).replace('\\', '/') with open(codeowners, 'a') as f: f.write(new_line) f.write('\n')
def _download_vendored_sdk(required_sdk, path): import tempfile import zipfile path_regex = re.compile( r'.*((\s*.*downloaded\s)|(\s*.*saved\s))(?P<path>.*\.whl)', re.IGNORECASE | re.S) temp_path = tempfile.mkdtemp() # download and extract the required SDK to the vendored_sdks folder downloaded_path = None if required_sdk: display('Downloading {}...'.format(required_sdk)) vendored_sdks_path = path result = pip_cmd('download {} --no-deps -d {}'.format( required_sdk, temp_path)).result try: result = result.decode('utf-8') except AttributeError: pass for line in result.splitlines(): try: downloaded_path = path_regex.match(line).group('path') except AttributeError: continue break if not downloaded_path: display('Unable to download') raise CLIError('Unable to download: {}'.format(required_sdk)) # extract the WHL file with zipfile.ZipFile(str(downloaded_path), 'r') as z: z.extractall(temp_path) _copy_vendored_sdk(temp_path, vendored_sdks_path)
def delete_groups(prefixes=None, older_than=6, product='azurecli', cause='automation', yes=False): from datetime import datetime, timedelta require_azure_cli() groups = json.loads(run_cmd('az group list -ojson').result) groups_to_delete = [] def _filter_by_tags(): for group in groups: group = Data(**group) if not group.tags: # pylint: disable=no-member continue tags = Data(**group.tags) # pylint: disable=no-member try: date_tag = datetime.strptime(tags.date, '%Y-%m-%dT%H:%M:%SZ') curr_time = datetime.utcnow() if (tags.product == product and tags.cause == cause and (curr_time - date_tag <= timedelta(hours=older_than + 1))): groups_to_delete.append(group.name) except AttributeError: continue def _filter_by_prefix(): for group in groups: group = Data(**group) for prefix in prefixes: if group.name.startswith(prefix): groups_to_delete.append(group.name) def _delete(): for group in groups_to_delete: run_cmd('az group delete -g {} -y --no-wait'.format(group), message=True) if prefixes: logger.info('Filter by prefix') _filter_by_prefix() else: logger.info('Filter by tags') _filter_by_tags() if not groups_to_delete: raise CLIError('No groups meet the criteria to delete.') if yes: _delete() else: subheading('Groups to Delete') for group in groups_to_delete: display('\t{}'.format(group)) if prompt_y_n('Delete {} resource groups?'.format(len(groups_to_delete)), 'y'): _delete() else: raise CLIError('Command cancelled.')
def add_ext_repo(path): try: _check_repo(path) except CLIError as ex: logger.error(ex) return False ext_repos.append(path) display('Repo {} OK.'.format(path)) return True
def __exit__(self, exc_type, exc_val, exc_tb): if self.target_profile is not None and self.target_profile != self.origin_profile: display('Switching back to origin profile "{}"...'.format( self.origin_profile)) call('az cloud update --profile {}'.format(self.origin_profile)) if exc_tb: display('') traceback.print_exception(exc_type, exc_val, exc_tb)
def __enter__(self): if self.target_profile is None or self.target_profile == self.origin_profile: display( 'The tests are set to run against current profile "{}"'.format( self.origin_profile)) else: result = cmd( 'az cloud update --profile {}'.format(self.target_profile), 'Switching to target profile "{}"...'.format( self.target_profile)) if result.exit_code != 0: raise CLIError(result.error.output.decode('utf-8'))
def generate_cli_ref_docs(output_dir=None, output_type=None): # require that azure cli installed require_azure_cli() output_dir = _process_ref_doc_output_dir(output_dir) heading('Generate CLI Reference Docs') display("Docs will be placed in {}.".format(output_dir)) # Generate documentation for all comamnds _call_sphinx_build(output_type, output_dir) display("\nThe {} files are in {}".format(output_type, output_dir))
def verify_versions(): import tempfile import shutil require_azure_cli() heading('Verify CLI Versions') path_table = get_path_table() modules = list(path_table['core'].items()) modules = [x for x in modules if x[0] not in EXCLUDED_MODULES] if not modules: raise CLIError('No modules selected to test.') display('MODULES: {}'.format(', '.join([x[0] for x in modules]))) results = {} original_cwd = os.getcwd() temp_dir = tempfile.mkdtemp() for mod, mod_path in modules: if not mod.startswith(COMMAND_MODULE_PREFIX) and mod != 'azure-cli': mod = '{}{}'.format(COMMAND_MODULE_PREFIX, mod) results[mod] = {} results.update( _compare_module_against_pypi(results, temp_dir, mod, mod_path)) shutil.rmtree(temp_dir) os.chdir(original_cwd) logger.info('Module'.ljust(40) + 'Local Version'.rjust(20) + 'Public Version'.rjust(20)) # pylint: disable=logging-not-lazy for mod, data in results.items(): logger.info( mod.ljust(40) + data['local_version'].rjust(20) + data['public_version'].rjust(20)) bump_mods = {k: v for k, v in results.items() if v['status'] == 'BUMP'} subheading('RESULTS') if bump_mods: logger.error( 'The following modules need their versions bumped. ' 'Scroll up for details: %s', ', '.join(bump_mods.keys())) logger.warning( '\nNote that before changing versions, you should consider ' 'running `git clean` to remove untracked files from your repo. ' 'Files that were once tracked but removed from the source may ' 'still be on your machine, resuling in false positives.') sys.exit(1) else: display('OK!')
def _generate_ref_docs_for_public_exts(output_type, base_output_dir): # TODO: this shouldn't define the env key, but should reference it from a central place in the cli repo. ENV_KEY_AZURE_EXTENSION_DIR = 'AZURE_EXTENSION_DIR' extensions_url_tups = _get_available_extension_urls() if not extensions_url_tups: raise CLIError("Failed to retrieve public extensions.") temp_dir = tempfile.mkdtemp(prefix="temp_whl_ext_dir") _logger.debug("Created temp directory to store downloaded whl files: %s", temp_dir) try: for name, file_name, download_url in extensions_url_tups: # for every compatible public extensions # download the whl file whl_file_name = _get_whl_from_url(download_url, file_name, temp_dir) # install the whl file in a new temp directory installed_ext_dir = tempfile.mkdtemp(prefix="temp_extension_dir_", dir=temp_dir) _logger.debug( "Created temp directory %s to use as the extension installation dir for %s extension.", installed_ext_dir, name) pip_cmd = [ sys.executable, '-m', 'pip', 'install', '--target', os.path.join(installed_ext_dir, 'extension'), whl_file_name, '--disable-pip-version-check', '--no-cache-dir' ] display('Executing "{}"'.format(' '.join(pip_cmd))) check_call(pip_cmd) # set the directory as the extension directory in the environment used to call sphinx-build env = os.environ.copy() env[ENV_KEY_AZURE_EXTENSION_DIR] = installed_ext_dir # generate documentation for installed extensions ext_output_dir = os.path.join(base_output_dir, name) os.makedirs(ext_output_dir) _call_sphinx_build(output_type, ext_output_dir, for_extensions_alone=True, call_env=env, msg="\nGenerating ref docs for {}".format(name)) finally: # finally delete the temp dir shutil.rmtree(temp_dir) _logger.debug("Deleted temp whl extension directory: %s", temp_dir)
def _run_test(test_args): cmd = ("python " + ('-B ' if test_args[1] else ' ') + "-m pytest {}").format(' '.join([test_args[0]] + test_args[2])) try: subprocess.check_call(cmd.split(), shell=const.IS_WINDOWS) except subprocess.CalledProcessError as e: if test_args[1]: display("Test failed, cleaning up recordings") recordings = os.path.join(test_args[0], 'recordings') if os.path.isdir(recordings): recording_files = os.listdir(recordings) for file in recording_files: if file.endswith(".yaml"): os.remove(os.path.join(recordings, file)) raise e
def _generate_extension(ext_name, repo_path, swagger_readme_file_path, use): heading('Start generating extension {}.'.format(ext_name)) # check if npm is installed try: shell_cmd('npm --version', stdout=subprocess.DEVNULL, raise_ex=False) except CLIError as ex: raise CLIError('{}\nPlease install npm.'.format(ex)) display('Installing autorest...\n') if const.IS_WINDOWS: try: shell_cmd('npm install -g autorest', raise_ex=False) except CLIError as ex: raise CLIError("Failed to install autorest.\n{}".format(ex)) else: try: shell_cmd('npm install -g autorest', stderr=subprocess.DEVNULL, raise_ex=False) except CLIError as ex: path = os.environ['PATH'] # check if npm is installed through nvm if os.environ.get('NVM_DIR'): raise ex # check if user using specific node version and manually add it to the os env PATH node_version = shell_cmd('node --version', capture_output=True).result if 'node/' + node_version + '/bin' in path: raise ex # create a new directory for npm global installations, to avoid using sudo in installing autorest npm_path = os.path.join(os.environ['HOME'], '.npm-packages') if not os.path.isdir(npm_path): os.mkdir(npm_path) npm_prefix = shell_cmd('npm prefix -g', capture_output=True).result shell_cmd('npm config set prefix ' + npm_path) os.environ['PATH'] = path + ':' + os.path.join(npm_path, 'bin') os.environ['MANPATH'] = os.path.join(npm_path, 'share', 'man') shell_cmd('npm install -g autorest') shell_cmd('npm config set prefix ' + npm_prefix) # update autorest core shell_cmd('autorest --latest') if not use: cmd = 'autorest --az --azure-cli-extension-folder={} {}'.format( repo_path, swagger_readme_file_path) else: cmd = 'autorest --az --azure-cli-extension-folder={} {} --use={}'.format( repo_path, swagger_readme_file_path, use) shell_cmd(cmd, message=True)
def check_license_headers(): heading('Verify License Headers') cli_path = get_cli_repo_path() all_paths = [cli_path] try: ext_repo = get_ext_repo_paths() for path in ext_repo: all_paths.append(path) except CLIError: display("No CLI ext path, running check only on modules") files_without_header = [] for path in all_paths: py_files = pathlib.Path(path).glob('**' + os.path.sep + '*.py') for py_file in py_files: py_file = str(py_file) if py_file.endswith('azure_bdist_wheel.py'): continue for ignore_token in _IGNORE_SUBDIRS: if ignore_token in py_file: break else: with open(str(py_file), 'r', encoding='utf-8') as f: file_text = f.read() if not file_text: continue test_results = [ LICENSE_HEADER in file_text, WRAPPED_LICENSE_HEADER in file_text, CODEGEN_LICENSE_HEADER in file_text ] if not any(test_results): files_without_header.append(py_file) subheading('Results') if files_without_header: raise CLIError( "{}\nError: {} files don't have the required license headers.". format('\n'.join(files_without_header), len(files_without_header))) display('License headers verified OK.')
def install_cli(cli_path, venv_path): src_path = os.path.join(cli_path, 'src') activate_path = (os.path.join(venv_path, 'Scripts', 'activate') if const.IS_WINDOWS else 'source ' + os.path.join(venv_path, const.UN_BIN, const.UN_ACTIVATE)) delimiter = ' && ' if const.IS_WINDOWS else '; ' executable = None if const.IS_WINDOWS else '/bin/bash' display("\nvenv activate path is " + str(activate_path)) shell_cmd(activate_path + delimiter + 'pip install --ignore-installed azure-common', stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, raise_ex=False, executable=executable) display("\nInstalling telemetry ") shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-telemetry'), stdout=subprocess.DEVNULL, raise_ex=False, stderr=subprocess.DEVNULL, executable=executable) display("\nInstalling core ") shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-core'), stdout=subprocess.DEVNULL, raise_ex=False, stderr=subprocess.DEVNULL, executable=executable) shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli-testsdk'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, raise_ex=False, executable=executable) display("\nInstalling cli ") shell_cmd(activate_path + delimiter + const.PIP_E_CMD + os.path.join(src_path, 'azure-cli'), raise_ex=False, executable=executable) req_file = 'requirements.py3.{}.txt'.format( platform.system().lower() if const.IS_WINDOWS else platform.system()) req_file = "{}/src/azure-cli/{}".format(cli_path, req_file) display("Installing " + req_file) shell_cmd(activate_path + delimiter + const.PIP_R_CMD + req_file, raise_ex=False, executable=executable)
def update_setup_py(pin=False): require_azure_cli() heading('Update azure-cli setup.py') path_table = get_path_table() azure_cli_path = path_table['core']['azure-cli'] azure_cli_setup_path = find_files(azure_cli_path, SETUP_PY_NAME)[0] modules = list(path_table['core'].items()) + list(path_table['mod'].items()) modules = [x for x in modules if x[0] not in EXCLUDED_MODULES] results = {mod[0]: {} for mod in modules} results = _get_module_versions(results, modules) _update_setup_py(results, azure_cli_setup_path, pin) display('OK!')
def _check_setup_py(results, update, pin): # only audit or update setup.py when all modules are being considered # otherwise, edge cases could arise if update is None: return results # retrieve current versions from azure-cli's setup.py file azure_cli_path = get_path_table(include_only='azure-cli')['core']['azure-cli'] azure_cli_setup_path = find_files(azure_cli_path, SETUP_PY_NAME)[0] with open(azure_cli_setup_path, 'r') as f: setup_py_version_regex = re.compile(r"(?P<quote>[\"'])(?P<mod>[^'=]*)(==(?P<ver>[\d.]*))?(?P=quote)") for line in f.readlines(): if line.strip().startswith("'azure-cli-"): match = setup_py_version_regex.match(line.strip()) mod = match.group('mod') if mod == 'azure-cli-command-modules-nspkg': mod = 'azure-cli-command_modules-nspkg' try: results[mod]['setup_version'] = match.group('ver') except KeyError: # something that is in setup.py but isn't module is # inherently a mismatch results[mod] = { 'local_version': 'Unavailable', 'public_version': 'Unknown', 'setup_version': match.group('ver'), 'status': 'MISMATCH' } if update: _update_setup_py(results, azure_cli_setup_path, pin) else: display('\nAuditing azure-cli setup.py against local module versions...') for mod, data in results.items(): if mod == 'azure-cli': continue setup_version = data['setup_version'] if not setup_version: logger.warning('The azure-cli setup.py file is not using pinned versions. Aborting audit.') break elif setup_version != data['local_version']: data['status'] = 'MISMATCH' return results
def generate_cli_ref_docs(output_dir=None, output_type=None, all_profiles=None): # require that azure cli installed and warn the users if extensions are installed. require_azure_cli() output_dir = _process_ref_doc_output_dir(output_dir) _warn_if_exts_installed() heading('Generate CLI Reference Docs') display("Docs will be placed in {}.".format(output_dir)) if all_profiles: # Generate documentation for all commands and for all CLI profiles _generate_ref_docs_for_all_profiles(output_type, output_dir) else: # Generate documentation for all comamnds _call_sphinx_build(output_type, output_dir) display("\nThe {} files are in {}".format(output_type, output_dir))
def _summarize_test_results(xml_path): import xml.etree.ElementTree as ElementTree subheading('Results') root = ElementTree.parse(xml_path).getroot() summary = { 'time': root.get('time'), 'tests': root.get('tests'), 'skips': root.get('skips'), 'failures': root.get('failures'), 'errors': root.get('errors') } display('Time: {time} sec\tTests: {tests}\tSkipped: {skips}\tFailures: {failures}\tErrors: {errors}'.format( **summary)) failed = [] for item in root.findall('testcase'): if item.findall('failure'): file_and_class = '.'.join(item.get('classname').split('.')[-2:]) failed.append('{}.{}'.format(file_and_class, item.get('name'))) if failed: subheading('FAILURES') for name in failed: display(name) display('')
def check_history(modules=None): # TODO: Does not work with extensions path_table = get_path_table(include_only=modules) selected_modules = list(path_table['core'].items()) + list(path_table['mod'].items()) heading('Verify History') module_names = sorted([name for name, _ in selected_modules]) display('Verifying README and HISTORY files for modules: {}'.format(' '.join(module_names))) failed_mods = [] for name, path in selected_modules: errors = _check_readme_render(path) if errors: failed_mods.append(name) subheading('{} errors'.format(name)) for error in errors: logger.error('%s\n', error) subheading('Results') if failed_mods: display('The following modules have invalid README/HISTORYs:') logger.error('\n'.join(failed_mods)) logger.warning('See above for the full warning/errors') logger.warning('note: Line numbers in the errors map to the long_description of your setup.py.') sys.exit(1) display('OK')
def _install_modules(): all_modules = list(get_path_table()['mod'].items()) failures = [] mod_num = 1 total_mods = len(all_modules) for name, path in all_modules: try: pip_cmd( "install -q -e {}".format(path), "Installing module `{}` ({}/{})...".format( name, mod_num, total_mods)) mod_num += 1 except CalledProcessError as err: # exit code is not zero failures.append("Failed to install {}. Error message: {}".format( name, err.output)) for f in failures: display(f) return not any(failures)