def update_extension(extension_name, index_url=None, pip_extra_index_urls=None, pip_proxy=None):
try:
ext = get_extension(extension_name)
cur_version = ext.get_version()
try:
download_url, ext_sha256 = resolve_from_index(extension_name, cur_version=cur_version, index_url=index_url)
except NoExtensionCandidatesError as err:
logger.debug(err)
raise CLIError("No updates available for '{}'. Use --debug for more information.".format(extension_name))
# Copy current version of extension to tmp directory in case we need to restore it after a failed install.
backup_dir = os.path.join(tempfile.mkdtemp(), extension_name)
extension_path = get_extension_path(extension_name)
logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir)
shutil.copytree(extension_path, backup_dir)
# Remove current version of the extension
shutil.rmtree(extension_path)
# Install newer version
try:
_add_whl_ext(download_url, ext_sha256=ext_sha256,
pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
logger.debug('Deleting backup of old extension at %s', backup_dir)
shutil.rmtree(backup_dir)
# This gets the metadata for the extension *after* the update
_augment_telemetry_with_ext_info(extension_name)
except Exception as err:
logger.error('An error occurred whilst updating.')
logger.error(err)
logger.debug('Copying %s to %s', backup_dir, extension_path)
shutil.copytree(backup_dir, extension_path)
raise CLIError('Failed to update. Rolled {} back to {}.'.format(extension_name, cur_version))
except ExtensionNotInstalledException as e:
raise CLIError(e)
def remove_extension(extension_name):
try:
# Get the extension and it will raise an error if it doesn't exist
get_extension(extension_name)
# We call this just before we remove the extension so we can get the metadata before it is gone
_augment_telemetry_with_ext_info(extension_name)
shutil.rmtree(get_extension_path(extension_name))
except ExtensionNotInstalledException as e:
raise CLIError(e)
def _add_whl_ext(source): # pylint: disable=too-many-statements
url_parse_result = urlparse(source)
is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
logger.debug('Extension source is url? %s', is_url)
whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
parsed_filename = WHEEL_INFO_RE(whl_filename)
extension_name = parsed_filename.groupdict().get('name') if parsed_filename else None
if not extension_name:
raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
if extension_exists(extension_name):
raise CLIError('The extension {} already exists.'.format(extension_name))
ext_file = None
if is_url:
# Download from URL
tmp_dir = tempfile.mkdtemp()
ext_file = os.path.join(tmp_dir, whl_filename)
logger.debug('Downloading %s to %s', source, ext_file)
try:
_whl_download_from_url(url_parse_result, ext_file)
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
logger.debug('Downloaded to %s', ext_file)
else:
# Get file path
ext_file = os.path.realpath(os.path.expanduser(source))
if not os.path.isfile(ext_file):
raise CLIError("File {} not found.".format(source))
# Validate the extension
logger.debug('Validating the extension {}'.format(ext_file))
try:
_validate_whl_extension(ext_file)
except AssertionError:
logger.debug(traceback.format_exc())
raise CLIError('The extension is invalid. Use --debug for more information.')
except CLIError as e:
raise e
logger.debug('Validation successful on {}'.format(ext_file))
# Install with pip
extension_path = get_extension_path(extension_name)
pip_args = ['install', '--target', extension_path, ext_file]
logger.debug('Executing pip with args: %s', pip_args)
with HomebrewPipPatch():
pip_status_code = _run_pip(pip_args)
if pip_status_code > 0:
logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
shutil.rmtree(extension_path, ignore_errors=True)
raise CLIError('An error occurred. Pip failed with status code {}. '
'Use --debug for more information.'.format(pip_status_code))
# Save the whl we used to install the extension in the extension dir.
dst = os.path.join(extension_path, whl_filename)
shutil.copyfile(ext_file, dst)
logger.debug('Saved the whl to %s', dst)
def remove_extension(extension_name):
def log_err(func, path, exc_info):
logger.debug("Error occurred attempting to delete item from the extension '%s'.", extension_name)
logger.debug("%s: %s - %s", func, path, exc_info)
try:
# Get the extension and it will raise an error if it doesn't exist
get_extension(extension_name)
# We call this just before we remove the extension so we can get the metadata before it is gone
_augment_telemetry_with_ext_info(extension_name)
shutil.rmtree(get_extension_path(extension_name), onerror=log_err)
except ExtensionNotInstalledException as e:
raise CLIError(e)
def remove_extension(extension_name):
def log_err(func, path, exc_info):
logger.debug("Error occurred attempting to delete item from the extension '%s'.", extension_name)
logger.debug("%s: %s - %s", func, path, exc_info)
try:
# Get the extension and it will raise an error if it doesn't exist
get_extension(extension_name)
# We call this just before we remove the extension so we can get the metadata before it is gone
_augment_telemetry_with_ext_info(extension_name)
shutil.rmtree(get_extension_path(extension_name), onerror=log_err)
except ExtensionNotInstalledException as e:
raise CLIError(e)
def remove_extension(extension_name):
def log_err(func, path, exc_info):
logger.debug("Error occurred attempting to delete item from the extension '%s'.", extension_name)
logger.debug("%s: %s - %s", func, path, exc_info)
try:
# Get the extension and it will raise an error if it doesn't exist
ext = get_extension(extension_name)
if ext and isinstance(ext, DevExtension):
raise CLIError(
"Extension '{name}' was installed in development mode. Remove using "
"`azdev extension remove {name}`".format(name=extension_name))
# We call this just before we remove the extension so we can get the metadata before it is gone
_augment_telemetry_with_ext_info(extension_name)
shutil.rmtree(get_extension_path(extension_name), onerror=log_err)
except ExtensionNotInstalledException as e:
raise CLIError(e)
def remove_extension(extension_name):
def log_err(func, path, exc_info):
logger.warning("Error occurred attempting to delete item from the extension '%s'.", extension_name)
logger.warning("%s: %s - %s", func, path, exc_info)
try:
# Get the extension and it will raise an error if it doesn't exist
ext = get_extension(extension_name)
if ext and isinstance(ext, DevExtension):
raise CLIError(
"Extension '{name}' was installed in development mode. Remove using "
"`azdev extension remove {name}`".format(name=extension_name))
# We call this just before we remove the extension so we can get the metadata before it is gone
_augment_telemetry_with_ext_info(extension_name)
shutil.rmtree(get_extension_path(extension_name), onerror=log_err)
except ExtensionNotInstalledException as e:
raise CLIError(e)
def _update_command_table_from_extensions(ext_suppressions):
def _handle_extension_suppressions(extensions):
filtered_extensions = []
for ext in extensions:
should_include = True
for suppression in ext_suppressions:
if should_include and suppression.handle_suppress(ext):
should_include = False
if should_include:
filtered_extensions.append(ext)
return filtered_extensions
extensions = get_extensions()
if extensions:
logger.debug("Found %s extensions: %s", len(extensions), [e.name for e in extensions])
allowed_extensions = _handle_extension_suppressions(extensions)
module_commands = set(self.command_table.keys())
for ext in allowed_extensions:
ext_name = ext.name
ext_dir = ext.path or get_extension_path(ext_name)
sys.path.append(ext_dir)
try:
ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir)
# Add to the map. This needs to happen before we load commands as registering a command
# from an extension requires this map to be up-to-date.
# self._mod_to_ext_map[ext_mod] = ext_name
start_time = timeit.default_timer()
extension_command_table, extension_group_table = \
_load_extension_command_loader(self, args, ext_mod)
for cmd_name, cmd in extension_command_table.items():
cmd.command_source = ExtensionCommandSource(
extension_name=ext_name,
overrides_command=cmd_name in module_commands,
preview=ext.preview)
self.command_table.update(extension_command_table)
self.command_group_table.update(extension_group_table)
elapsed_time = timeit.default_timer() - start_time
logger.debug("Loaded extension '%s' in %.3f seconds.", ext_name, elapsed_time)
except Exception: # pylint: disable=broad-except
self.cli_ctx.raise_event(EVENT_FAILED_EXTENSION_LOAD, extension_name=ext_name)
logger.warning("Unable to load extension '%s'. Use --debug for more information.", ext_name)
logger.debug(traceback.format_exc())
def get_extension_modules():
from importlib import import_module
import pkgutil
from azure.cli.core.extension import get_extensions, get_extension_path, get_extension_modname
extension_whls = get_extensions()
ext_modules = []
if extension_whls:
for ext_name in [e.name for e in extension_whls]:
ext_dir = get_extension_path(ext_name)
sys.path.append(ext_dir)
try:
ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir)
module = import_module(ext_mod)
setattr(module, 'path', module.__path__[0])
ext_modules.append((module, ext_mod))
except Exception as ex:
display("Error importing '{}' extension: {}".format(ext_mod, ex))
return ext_modules
def _get_command_table_from_extensions():
extensions = get_extension_names()
if extensions:
logger.debug("Found {} extensions: {}".format(len(extensions), extensions))
for ext_name in extensions:
ext_dir = get_extension_path(ext_name)
sys.path.append(ext_dir)
try:
ext_mod = get_extension_modname(ext_dir=ext_dir)
# Add to the map. This needs to happen before we load commands as registering a command
# from an extension requires this map to be up-to-date.
mod_to_ext_map[ext_mod] = ext_name
start_time = timeit.default_timer()
import_module(ext_mod).load_commands()
elapsed_time = timeit.default_timer() - start_time
logger.debug("Loaded extension '%s' in %.3f seconds.", ext_name, elapsed_time)
except Exception: # pylint: disable=broad-except
logger.warning("Unable to load extension '%s'. Use --debug for more information.", ext_name)
logger.debug(traceback.format_exc())
def update_cmd_tree(ext_name):
print(f"Processing {ext_name}")
ext_dir = get_extension_path(ext_name)
ext_mod = get_extension_modname(ext_name, ext_dir=ext_dir)
invoker = az_cli.invocation_cls(
cli_ctx=az_cli,
commands_loader_cls=az_cli.commands_loader_cls,
parser_cls=az_cli.parser_cls,
help_cls=az_cli.help_cls)
az_cli.invocation = invoker
sys.path.append(ext_dir)
extension_command_table, _ = _load_extension_command_loader(
invoker.commands_loader, None, ext_mod)
EXT_CMD_TREE_TO_UPLOAD = Session()
EXT_CMD_TREE_TO_UPLOAD.load(
os.path.expanduser(os.path.join('~', '.azure', file_name)))
root = {}
for cmd_name, ext_cmd in extension_command_table.items():
try:
# do not include hidden deprecated command
if ext_cmd.deprecate_info.hide:
print(f"Skip hidden deprecated command: {cmd_name}")
continue
except AttributeError:
pass
parts = cmd_name.split()
parent = root
for i, part in enumerate(parts):
if part in parent:
pass
elif i == len(parts) - 1:
parent[part] = ext_name
else:
parent[part] = {}
parent = parent[part]
print(root)
for k, v in root.items():
merge(EXT_CMD_TREE_TO_UPLOAD.data, k, v)
EXT_CMD_TREE_TO_UPLOAD.save_with_retry()
def _get_command_table_from_extensions():
extensions = get_extension_names()
if extensions:
logger.debug("Found {} extensions: {}".format(len(extensions), extensions))
for ext_name in extensions:
ext_dir = get_extension_path(ext_name)
sys.path.append(ext_dir)
try:
ext_mod = get_extension_modname(ext_dir=ext_dir)
# Add to the map. This needs to happen before we load commands as registering a command
# from an extension requires this map to be up-to-date.
mod_to_ext_map[ext_mod] = ext_name
start_time = timeit.default_timer()
import_module(ext_mod).load_commands()
elapsed_time = timeit.default_timer() - start_time
logger.debug("Loaded extension '%s' in %.3f seconds.", ext_name, elapsed_time)
except Exception: # pylint: disable=broad-except
logger.warning("Unable to load extension '%s'. Use --debug for more information.", ext_name)
logger.debug(traceback.format_exc())
def ensure_azure_namespace_path():
"""
Run prior to importing azure namespace packages (azure.*) to ensure the
extension root path is configured for package import.
"""
from azure.cli.core.extension import get_extension_path
from azext_iot.constants import EXTENSION_NAME
ext_path = get_extension_path(EXTENSION_NAME)
ext_azure_dir = os.path.join(ext_path, "azure")
if os.path.isdir(ext_azure_dir):
import azure
if getattr(azure, "__path__", None) and ext_azure_dir not in azure.__path__:
azure.__path__.append(ext_azure_dir) # _NamespacePath /w PEP420
if sys.path and sys.path[0] != ext_path:
sys.path.insert(0, ext_path)
return
def install(package, exact_version=None, compatible_version=None):
if not extension_exists(EXTENSION_NAME):
raise RuntimeError('iot extension is misconfigured')
ext_path = get_extension_path(EXTENSION_NAME)
cmd = [sys.executable, '-m', 'pip', '--disable-pip-version-check', '--no-cache-dir', 'install', '-U', '--target', ext_path]
cmd_suffix = None
if exact_version:
cmd_suffix = '{}=={}'.format(package, exact_version)
elif compatible_version:
cmd_suffix = '{}~={}'.format(package, compatible_version)
else:
cmd_suffix = package
cmd.append(cmd_suffix)
logger.info(cmd)
try:
log_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True)
logger.debug(log_output)
return True
except subprocess.CalledProcessError as e:
logger.debug(e.output)
logger.debug(e)
return False
def _add_whl_ext(source, ext_sha256=None, pip_extra_index_urls=None, pip_proxy=None): # pylint: disable=too-many-statements
if not source.endswith('.whl'):
raise ValueError('Unknown extension type. Only Python wheels are supported.')
url_parse_result = urlparse(source)
is_url = (url_parse_result.scheme == 'http' or url_parse_result.scheme == 'https')
logger.debug('Extension source is url? %s', is_url)
whl_filename = os.path.basename(url_parse_result.path) if is_url else os.path.basename(source)
parsed_filename = WHEEL_INFO_RE(whl_filename)
# Extension names can have - but .whl format changes it to _ (PEP 0427). Undo this.
extension_name = parsed_filename.groupdict().get('name').replace('_', '-') if parsed_filename else None
if not extension_name:
raise CLIError('Unable to determine extension name from {}. Is the file name correct?'.format(source))
if extension_exists(extension_name):
raise CLIError('The extension {} already exists.'.format(extension_name))
ext_file = None
if is_url:
# Download from URL
tmp_dir = tempfile.mkdtemp()
ext_file = os.path.join(tmp_dir, whl_filename)
logger.debug('Downloading %s to %s', source, ext_file)
try:
_whl_download_from_url(url_parse_result, ext_file)
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
raise CLIError('Please ensure you have network connection. Error detail: {}'.format(str(err)))
logger.debug('Downloaded to %s', ext_file)
else:
# Get file path
ext_file = os.path.realpath(os.path.expanduser(source))
if not os.path.isfile(ext_file):
raise CLIError("File {} not found.".format(source))
# Validate the extension
logger.debug('Validating the extension %s', ext_file)
if ext_sha256:
valid_checksum, computed_checksum = is_valid_sha256sum(ext_file, ext_sha256)
if valid_checksum:
logger.debug("Checksum of %s is OK", ext_file)
else:
logger.debug("Invalid checksum for %s. Expected '%s', computed '%s'.",
ext_file, ext_sha256, computed_checksum)
raise CLIError("The checksum of the extension does not match the expected value. "
"Use --debug for more information.")
try:
_validate_whl_extension(ext_file)
except AssertionError:
logger.debug(traceback.format_exc())
raise CLIError('The extension is invalid. Use --debug for more information.')
except CLIError as e:
raise e
logger.debug('Validation successful on %s', ext_file)
# Check for distro consistency
check_distro_consistency()
# Install with pip
extension_path = get_extension_path(extension_name)
pip_args = ['install', '--target', extension_path, ext_file]
if pip_proxy:
pip_args = pip_args + ['--proxy', pip_proxy]
if pip_extra_index_urls:
for extra_index_url in pip_extra_index_urls:
pip_args = pip_args + ['--extra-index-url', extra_index_url]
logger.debug('Executing pip with args: %s', pip_args)
with HomebrewPipPatch():
pip_status_code = _run_pip(pip_args)
if pip_status_code > 0:
logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
shutil.rmtree(extension_path, ignore_errors=True)
raise CLIError('An error occurred. Pip failed with status code {}. '
'Use --debug for more information.'.format(pip_status_code))
# Save the whl we used to install the extension in the extension dir.
dst = os.path.join(extension_path, whl_filename)
shutil.copyfile(ext_file, dst)
logger.debug('Saved the whl to %s', dst)
def remove_extension(extension_name):
try:
get_extension(extension_name)
shutil.rmtree(get_extension_path(extension_name))
except ExtensionNotInstalledException as e:
raise CLIError(e)
def _install_test_extension2(): # pylint: disable=no-self-use
# We extract the extension into place as we aren't testing install here
zip_file = _get_test_data_file('myfirstcliextension_az_extmetadata.zip')
zip_ref = zipfile.ZipFile(zip_file, 'r')
zip_ref.extractall(get_extension_path(EXT_NAME))
zip_ref.close()
def _install_test_extension1(): # pylint: disable=no-self-use
# We extract the extension into place as we aren't testing install here
zip_file = _get_test_data_file('{}.zip'.format(EXT_NAME))
zip_ref = zipfile.ZipFile(zip_file, 'r')
zip_ref.extractall(get_extension_path(EXT_NAME))
zip_ref.close()
def add_extension_to_path(extension_name):
ext_dir = get_extension_path(extension_name)
sys.path.append(ext_dir)
