def remove_extension(extensions):
ext_paths = get_ext_repo_paths()
installed_paths = find_files(ext_paths, '*.*-info')
paths_to_remove = []
names_to_remove = []
if extensions == ['*']:
paths_to_remove = [os.path.dirname(path) for path in installed_paths]
names_to_remove = [
os.path.basename(os.path.dirname(path)) for path in installed_paths
]
else:
for path in installed_paths:
folder = os.path.dirname(path)
long_name = os.path.basename(folder)
if long_name in extensions:
paths_to_remove.append(folder)
names_to_remove.append(long_name)
extensions.remove(long_name)
# raise error if any extension not installed
if extensions:
raise CLIError('extension(s) not installed: {}'.format(
' '.join(extensions)))
# removes any links that may have been added to site-packages.
for ext in names_to_remove:
pip_cmd('uninstall {} -y'.format(ext))
for path in paths_to_remove:
for d in os.listdir(path):
# delete the egg-info and dist-info folders to make the extension invisible to the CLI and azdev
if d.endswith('egg-info') or d.endswith('dist-info'):
path_to_remove = os.path.join(path, d)
display("Removing '{}'...".format(path_to_remove))
shutil.rmtree(path_to_remove)
def install_draft_sdk(modules, private=False):
for module in modules:
kwargs = {
'module': module,
'pr': 'pr' if private else '',
'branch': 'restapi_auto_{}/resource-manager'.format(module)
}
pip_cmd(
'install "git+https://github.com/Azure/azure-sdk-for-python{pr}@{branch}'
'#egg=azure-mgmt-{module}&subdirectory=azure-mgmt-{module}"'.
format(**kwargs),
show_stderr=True,
message='Installing draft SDK for azure-mgmt-{}...'.format(module))
def _download_vendored_sdk(required_sdk, path):
import tempfile
import zipfile
path_regex = re.compile(
r'.*((\s*.*downloaded\s)|(\s*.*saved\s))(?P<path>.*\.whl)',
re.IGNORECASE | re.S)
temp_path = tempfile.mkdtemp()
# download and extract the required SDK to the vendored_sdks folder
downloaded_path = None
if required_sdk:
display('Downloading {}...'.format(required_sdk))
vendored_sdks_path = path
result = pip_cmd('download {} --no-deps -d {}'.format(
required_sdk, temp_path)).result
try:
result = result.decode('utf-8')
except AttributeError:
pass
for line in result.splitlines():
try:
downloaded_path = path_regex.match(line).group('path')
except AttributeError:
continue
break
if not downloaded_path:
display('Unable to download')
raise CLIError('Unable to download: {}'.format(required_sdk))
# extract the WHL file
with zipfile.ZipFile(str(downloaded_path), 'r') as z:
z.extractall(temp_path)
_copy_vendored_sdk(temp_path, vendored_sdks_path)
def add_extension(extensions):
ext_paths = get_ext_repo_paths()
all_extensions = find_files(ext_paths, 'setup.py')
if extensions == ['*']:
paths_to_add = [
os.path.dirname(path) for path in all_extensions
if 'site-packages' not in path
]
else:
paths_to_add = []
for path in all_extensions:
folder = os.path.dirname(path)
long_name = os.path.basename(folder)
if long_name in extensions:
paths_to_add.append(folder)
extensions.remove(long_name)
# raise error if any extension wasn't found
if extensions:
raise CLIError('extension(s) not found: {}'.format(
' '.join(extensions)))
for path in paths_to_add:
result = pip_cmd('install -e {}'.format(path),
"Adding extension '{}'...".format(path))
if result.error:
raise result.error # pylint: disable=raising-bad-type
def _install_extensions(ext_paths):
# clear pre-existing dev extensions
installed_extensions = [
x['name'] for x in list_extensions() if x['inst'] == 'Y'
]
remove_extension(installed_extensions)
# install specified extensions
for path in ext_paths or []:
result = pip_cmd('install -e {}'.format(path),
"Adding extension '{}'...".format(path))
if result.error:
raise result.error # pylint: disable=raising-bad-type
def _install_extensions(ext_paths):
# clear pre-existing dev extensions
try:
installed_extensions = [x['name'] for x in list_extensions() if x['install'] == 'Y']
remove_extension(installed_extensions)
except KeyError as ex:
logger.warning('Error occurred determining installed extensions. Run with --debug for more info.')
logger.debug(ex)
# install specified extensions
for path in ext_paths or []:
result = pip_cmd('install -e {}'.format(path), "Adding extension '{}'...".format(path))
if result.error:
raise result.error # pylint: disable=raising-bad-type
def _install_modules():
all_modules = list(get_path_table()['mod'].items())
failures = []
mod_num = 1
total_mods = len(all_modules)
for name, path in all_modules:
try:
pip_cmd(
"install -q -e {}".format(path),
"Installing module `{}` ({}/{})...".format(
name, mod_num, total_mods))
mod_num += 1
except CalledProcessError as err:
# exit code is not zero
failures.append("Failed to install {}. Error message: {}".format(
name, err.output))
for f in failures:
display(f)
return not any(failures)
def _install_cli(cli_path, deps=None):
if not cli_path:
# install public CLI off PyPI if no repo found
pip_cmd('install --upgrade azure-cli', "Installing `azure-cli`...")
pip_cmd(
'install git+https://github.com/Azure/azure-cli@master#subdirectory=src/azure-cli-testsdk',
"Installing `azure-cli-testsdk`...")
return
if cli_path == 'EDGE':
# install the public edge build
pip_cmd(
'install --pre azure-cli --extra-index-url https://azurecliprod.blob.core.windows.net/edge',
"Installing `azure-cli` edge build...")
pip_cmd(
'install git+https://github.com/Azure/azure-cli@master#subdirectory=src/azure-cli-testsdk',
"Installing `azure-cli-testsdk`...")
return
# otherwise editable install from source
# install private whls if there are any
privates_dir = os.path.join(cli_path, "privates")
if os.path.isdir(privates_dir) and os.listdir(privates_dir):
whl_list = " ".join(
[os.path.join(privates_dir, f) for f in os.listdir(privates_dir)])
pip_cmd("install -q {}".format(whl_list),
"Installing private whl files...")
# install general requirements
pip_cmd("install -q -r {}/requirements.txt".format(cli_path),
"Installing `requirements.txt`...")
if deps == 'setup.py':
# Resolve dependencies from setup.py files.
# command modules have dependency on azure-cli-core so install this first
pip_cmd("install -q -e {}/src/azure-cli-telemetry".format(cli_path),
"Installing `azure-cli-telemetry`...")
pip_cmd("install -q -e {}/src/azure-cli-core".format(cli_path),
"Installing `azure-cli-core`...")
# azure cli has dependencies on the above packages so install this one last
pip_cmd("install -q -e {}/src/azure-cli".format(cli_path),
"Installing `azure-cli`...")
pip_cmd("install -q -e {}/src/azure-cli-testsdk".format(cli_path),
"Installing `azure-cli-testsdk`...")
else:
# First install packages without dependencies,
# then resolve dependencies from requirements.*.txt file.
pip_cmd(
"install -e {}/src/azure-cli-telemetry --no-deps".format(cli_path),
"Installing `azure-cli-telemetry`...")
pip_cmd("install -e {}/src/azure-cli-core --no-deps".format(cli_path),
"Installing `azure-cli-core`...")
pip_cmd("install -e {}/src/azure-cli --no-deps".format(cli_path),
"Installing `azure-cli`...")
# The dependencies of testsdk are not in requirements.txt as this package is not needed by the
# azure-cli package for running commands.
# Here we need to install with dependencies for azdev test.
pip_cmd("install -e {}/src/azure-cli-testsdk".format(cli_path),
"Installing `azure-cli-testsdk`...")
import platform
system = platform.system()
req_file = 'requirements.py3.{}.txt'.format(system)
pip_cmd("install -r {}/src/azure-cli/{}".format(cli_path, req_file),
"Installing `{}`...".format(req_file))
def setup(cli_path=None, ext_repo_path=None, ext=None, deps=None):
require_virtual_env()
start = time.time()
heading('Azure CLI Dev Setup')
ext_to_install = []
if not any([cli_path, ext_repo_path, ext]):
cli_path, ext_repo_path, ext_to_install = _interactive_setup()
else:
if cli_path == "pypi":
cli_path = None
# otherwise assume programmatic setup
if cli_path:
CLI_SENTINEL = 'azure-cli.pyproj'
if cli_path == Flag:
cli_path = find_file(CLI_SENTINEL)
if not cli_path:
raise CLIError(
'Unable to locate your CLI repo. Things to check:'
'\n Ensure you have cloned the repo. '
'\n Specify the path explicitly with `-c PATH`. '
'\n If you run with `-c` to autodetect, ensure you are running '
'this command from a folder upstream of the repo.')
if cli_path != 'EDGE':
cli_path = _check_path(cli_path, CLI_SENTINEL)
display('Azure CLI:\n {}\n'.format(cli_path))
else:
display('Azure CLI:\n PyPI\n')
# must add the necessary repo to add an extension
if ext and not ext_repo_path:
raise CLIError(
'usage error: --repo EXT_REPO [EXT_REPO ...] [--ext EXT_NAME ...]'
)
get_azure_config().set_value('extension', 'dev_sources', '')
if ext_repo_path:
# add extension repo(s)
add_extension_repo(ext_repo_path)
display('Azure CLI extension repos:\n {}'.format('\n '.join(
[os.path.abspath(x) for x in ext_repo_path])))
if ext == ['*']:
ext_to_install = [x['path'] for x in list_extensions()]
elif ext:
# add extension(s)
available_extensions = [x['name'] for x in list_extensions()]
not_found = [x for x in ext if x not in available_extensions]
if not_found:
raise CLIError(
"The following extensions were not found. Ensure you have added "
"the repo using `--repo/-r PATH`.\n {}".format(
'\n '.join(not_found)))
ext_to_install = [
x['path'] for x in list_extensions() if x['name'] in ext
]
if ext_to_install:
display('\nAzure CLI extensions:\n {}'.format(
'\n '.join(ext_to_install)))
dev_sources = get_azure_config().get('extension', 'dev_sources', None)
# save data to config files
config = get_azdev_config()
config.set_value('ext', 'repo_paths',
dev_sources if dev_sources else '_NONE_')
config.set_value('cli', 'repo_path', cli_path if cli_path else '_NONE_')
# install packages
subheading('Installing packages')
# upgrade to latest pip
pip_cmd('install --upgrade pip -q', 'Upgrading pip...')
_install_cli(cli_path, deps=deps)
_install_extensions(ext_to_install)
_copy_config_files()
end = time.time()
elapsed_min = int((end - start) / 60)
elapsed_sec = int(end - start) % 60
display('\nElapsed time: {} min {} sec'.format(elapsed_min, elapsed_sec))
subheading('Finished dev setup!')
def _create_package(prefix,
repo_path,
is_ext,
name='test',
display_name=None,
display_name_plural=None,
required_sdk=None,
client_name=None,
operation_name=None,
sdk_property=None,
not_preview=False,
local_sdk=None):
from jinja2 import Environment, PackageLoader
if local_sdk and required_sdk:
raise CLIError(
'usage error: --local-sdk PATH | --required-sdk NAME==VER')
if name.startswith(prefix):
name = name[len(prefix):]
heading('Create CLI {}: {}{}'.format('Extension' if is_ext else 'Module',
prefix, name))
# package_name is how the item should show up in `pip list`
package_name = '{}{}'.format(prefix, name.replace(
'_', '-')) if not is_ext else name
display_name = display_name or name.capitalize()
kwargs = {
'name':
name,
'mod_path':
'{}{}'.format(prefix, name)
if is_ext else 'azure.cli.command_modules.{}'.format(name),
'display_name':
display_name,
'display_name_plural':
display_name_plural or '{}s'.format(display_name),
'loader_name':
'{}CommandsLoader'.format(name.capitalize()),
'pkg_name':
package_name,
'ext_long_name':
'{}{}'.format(prefix, name) if is_ext else None,
'is_ext':
is_ext,
'is_preview':
not not_preview
}
new_package_path = os.path.join(repo_path, package_name)
if os.path.isdir(new_package_path):
if not prompt_y_n("{} '{}' already exists. Overwrite?".format(
'Extension' if is_ext else 'Module', package_name),
default='n'):
raise CLIError('aborted by user')
ext_folder = '{}{}'.format(prefix, name) if is_ext else None
# create folder tree
if is_ext:
_ensure_dir(
os.path.join(new_package_path, ext_folder, 'tests', 'latest'))
_ensure_dir(os.path.join(new_package_path, ext_folder,
'vendored_sdks'))
else:
_ensure_dir(os.path.join(new_package_path, 'tests', 'latest'))
env = Environment(loader=PackageLoader('azdev', 'mod_templates'))
# determine dependencies
dependencies = []
if is_ext:
if required_sdk:
_download_vendored_sdk(required_sdk,
path=os.path.join(new_package_path,
ext_folder,
'vendored_sdks'))
elif local_sdk:
_copy_vendored_sdk(
local_sdk,
os.path.join(new_package_path, ext_folder, 'vendored_sdks'))
sdk_path = None
if any([local_sdk, required_sdk]):
sdk_path = '{}{}.vendored_sdks'.format(prefix, package_name)
kwargs.update({
'sdk_path': sdk_path,
'client_name': client_name,
'operation_name': operation_name,
'sdk_property': sdk_property or '{}_name'.format(name)
})
else:
if required_sdk:
version_regex = r'(?P<name>[a-zA-Z-]+)(?P<op>[~<>=]*)(?P<version>[\d.]*)'
version_comps = re.compile(version_regex).match(required_sdk)
sdk_kwargs = version_comps.groupdict()
kwargs.update({
'sdk_path': sdk_kwargs['name'].replace('-', '.'),
'client_name': client_name,
'operation_name': operation_name,
})
dependencies.append("'{}'".format(required_sdk))
else:
dependencies.append('# TODO: azure-mgmt-<NAME>==<VERSION>')
kwargs.update({'sdk_property': sdk_property or '{}_name'.format(name)})
kwargs['dependencies'] = dependencies
# generate code for root level
dest_path = new_package_path
if is_ext:
root_files = ['HISTORY.rst', 'README.rst', 'setup.cfg', 'setup.py']
_generate_files(env, kwargs, root_files, dest_path)
dest_path = dest_path if not is_ext else os.path.join(
dest_path, ext_folder)
module_files = [{
'name': '__init__.py',
'template': 'module__init__.py'
}, '_client_factory.py', '_help.py', '_params.py', '_validators.py',
'commands.py', 'custom.py']
if is_ext:
module_files.append('azext_metadata.json')
_generate_files(env, kwargs, module_files, dest_path)
dest_path = os.path.join(dest_path, 'tests')
blank_init = {'name': '__init__.py', 'template': 'blank__init__.py'}
_generate_files(env, kwargs, blank_init, dest_path)
dest_path = os.path.join(dest_path, 'latest')
test_files = [
blank_init, {
'name': 'test_{}_scenario.py'.format(name),
'template': 'test_service_scenario.py'
}
]
_generate_files(env, kwargs, test_files, dest_path)
if is_ext:
result = pip_cmd('install -e {}'.format(new_package_path),
"Installing `{}{}`...".format(prefix, name))
if result.error:
raise result.error # pylint: disable=raising-bad-type
def _compare_module_against_pypi(results, root_dir, mod, mod_path):
import zipfile
version_pattern = re.compile(r'.*azure_cli[^-]*-(\d*.\d*.\d*).*')
downloaded_path = None
downloaded_version = None
build_path = None
build_version = None
build_dir = os.path.join(root_dir, mod, 'local')
pypi_dir = os.path.join(root_dir, mod, 'public')
# download the public PyPI package and extract the version
logger.info('Checking %s...', mod)
result = pip_cmd('download {} --no-deps -d {}'.format(mod, root_dir)).result
try:
result = result.decode('utf-8')
except AttributeError:
pass
for line in result.splitlines():
line = line.strip()
if line.endswith('.whl') and line.startswith('Saved'):
downloaded_path = line.replace('Saved ', '').strip()
downloaded_version = version_pattern.match(downloaded_path).group(1)
break
if line.startswith('No matching distribution found'):
downloaded_path = None
downloaded_version = 'Unavailable'
break
if not downloaded_version:
raise CLIError('Unexpected error trying to acquire {}: {}'.format(mod, result))
# build from source and extract the version
setup_path = os.path.normpath(mod_path.strip())
os.chdir(setup_path)
py_cmd('setup.py bdist_wheel -d {}'.format(build_dir))
if len(os.listdir(build_dir)) != 1:
raise CLIError('Unexpectedly found multiple build files found in {}.'.format(build_dir))
build_path = os.path.join(build_dir, os.listdir(build_dir)[0])
build_version = version_pattern.match(build_path).group(1)
results[mod].update({
'local_version': build_version,
'public_version': downloaded_version
})
# OK if package is new
if downloaded_version == 'Unavailable':
results[mod]['status'] = 'OK'
return results
# OK if local version is higher than what's on PyPI
if LooseVersion(build_version) > LooseVersion(downloaded_version):
results[mod]['status'] = 'OK'
return results
# slight difference in dist-info dirs, so we must extract the azure folders and compare them
with zipfile.ZipFile(str(downloaded_path), 'r') as z:
z.extractall(pypi_dir)
with zipfile.ZipFile(str(build_path), 'r') as z:
z.extractall(build_dir)
errors = _compare_folders(os.path.join(pypi_dir), os.path.join(build_dir))
# clean up empty strings
errors = [e for e in errors if e]
if errors:
subheading('Differences found in {}'.format(mod))
for error in errors:
logger.warning(error)
results[mod]['status'] = 'OK' if not errors else 'BUMP'
# special case: to make a release, these MUST be bumped, even if it wouldn't otherwise be necessary
if mod in ['azure-cli', 'azure-cli-core']:
if results[mod]['status'] == 'OK':
logger.warning('%s version must be bumped to support release!', mod)
results[mod]['status'] = 'BUMP'
return results
def _install_cli(cli_path):
# install public CLI off PyPI if no repo found
if not cli_path:
pip_cmd('install --upgrade azure-cli', "Installing `azure-cli`...")
pip_cmd(
'install git+https://github.com/Azure/azure-cli@master#subdirectory=src/azure-cli-testsdk',
"Installing `azure-cli-testsdk`...")
return
# otherwise editable install from source
# install private whls if there are any
privates_dir = os.path.join(cli_path, "privates")
if os.path.isdir(privates_dir) and os.listdir(privates_dir):
whl_list = " ".join(
[os.path.join(privates_dir, f) for f in os.listdir(privates_dir)])
pip_cmd("install -q {}".format(whl_list),
"Installing private whl files...")
# install general requirements
pip_cmd("install -q -r {}/requirements.txt".format(cli_path),
"Installing `requirements.txt`...")
# command modules have dependency on azure-cli-core so install this first
pip_cmd("install -q -e {}/src/azure-cli-nspkg".format(cli_path),
"Installing `azure-cli-nspkg`...")
pip_cmd("install -q -e {}/src/azure-cli-telemetry".format(cli_path),
"Installing `azure-cli-telemetry`...")
pip_cmd("install -q -e {}/src/azure-cli-core".format(cli_path),
"Installing `azure-cli-core`...")
# azure cli has dependencies on the above packages so install this one last
pip_cmd("install -q -e {}/src/azure-cli".format(cli_path),
"Installing `azure-cli`...")
pip_cmd("install -q -e {}/src/azure-cli-testsdk".format(cli_path),
"Installing `azure-cli-testsdk`...")
# Ensure that the site package's azure/__init__.py has the old style namespace
# package declaration by installing the old namespace package
pip_cmd("install -q -I azure-nspkg==1.0.0", "Installing `azure-nspkg`...")
pip_cmd("install -q -I azure-mgmt-nspkg==1.0.0",
"Installing `azure-mgmt-nspkg`...")
def _add_extension(ext_name, repo_path):
new_package_path = os.path.join(repo_path, 'src', ext_name)
result = pip_cmd('install -e {}'.format(new_package_path),
"Adding extension `{}`...".format(new_package_path))
if result.error:
raise result.error