def validator(namespace): type_field_name = '{}_type'.format(property_name) property_val = getattr(namespace, property_name, None) parent_val = getattr(namespace, parent_name, None) if parent_name else None # Check for the different scenarios (order matters) # 1) provided value indicates None (pair of empty quotes) if property_val in ('', '""', "''") or (property_val is None and default_none): if not allow_none: raise CLIError('{} cannot be None.'.format(property_option)) setattr(namespace, type_field_name, 'none') setattr(namespace, property_name, None) if parent_name and parent_val: logger = azlogging.get_az_logger(__name__) logger.warning('Ignoring: %s %s', parent_option, parent_val) setattr(namespace, parent_name, None) return # SUCCESS # Create a resource ID we can check for existence. (resource_id_parts, value_was_id) = _validate_name_or_id( namespace.resource_group_name, property_val, property_type, parent_val, parent_type) # 2) resource exists if resource_exists(**resource_id_parts): setattr(namespace, type_field_name, 'existingId') setattr(namespace, property_name, resource_id(**resource_id_parts)) if parent_val: if value_was_id: logger = azlogging.get_az_logger(__name__) logger.warning('Ignoring: %s %s', parent_option, parent_val) setattr(namespace, parent_name, None) return # SUCCESS # if a parent name was required but not specified, raise a usage error if has_parent and not value_was_id and not parent_val and not allow_new: raise ValueError('incorrect usage: {0} ID | {0} NAME {1} NAME'.format( property_option, parent_option)) # if non-existent ID was supplied, throw error depending on whether a new resource can # be created. if value_was_id: usage_message = '{} NAME'.format(property_option) if not has_parent \ else '{} NAME [{} NAME]'.format(property_option, parent_option) action_message = 'Specify ( {} ) to create a new resource.'.format(usage_message) if \ allow_new else 'Create the required resource and try again.' raise CLIError('{} {} does not exist. {}'.format( property_name, property_val, action_message)) # 3) try to create new resource if allow_new: setattr(namespace, type_field_name, 'new') else: raise CLIError( '{} {} does not exist. Create the required resource and try again.'.format( property_name, property_val))
def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False): """ Download blobs in a container recursively :param str source: The string represents the source of this download operation. The source can be the container URL or the container name. When the source is the container URL, the storage account name will parsed from the URL. :param str destination: The string represents the destination folder of this download operation. The folder must exist. :param bool dryrun: Show the summary of the operations to be taken instead of actually download the file(s) :param str pattern: The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]', and '[!seq]'. """ source_blobs = list(collect_blobs(client, source_container_name, pattern)) if dryrun: logger = get_az_logger(__name__) logger.warning('download action: from %s to %s', source, destination) logger.warning(' pattern %s', pattern) logger.warning(' container %s', source_container_name) logger.warning(' total %d', len(source_blobs)) logger.warning(' operations') for b in source_blobs or []: logger.warning(' - %s', b) return [] return list(_download_blob(client, source_container_name, destination, blob) for blob in source_blobs)
def storage_file_upload_batch(client, destination, source, pattern=None, dryrun=False, validate_content=False, content_settings=None, max_connections=1, metadata=None): """ Upload local files to Azure Storage File Share in batch """ from .util import glob_files_locally source_files = [c for c in glob_files_locally(source, pattern)] if dryrun: logger = get_az_logger(__name__) logger.warning('upload files to file share') logger.warning(' account %s', client.account_name) logger.warning(' share %s', destination) logger.warning(' total %d', len(source_files or [])) logger.warning(' operations') for f in source_files or []: logger.warning(' - %s => %s', *f) return [] # TODO: Performance improvement # 1. Upload files in parallel def _upload_action(source_pair): dir_name = os.path.dirname(source_pair[1]) file_name = os.path.basename(source_pair[1]) _make_directory_in_files_share(client, destination, dir_name) create_file_args = { 'share_name': destination, 'directory_name': dir_name, 'file_name': file_name, 'local_file_path': source_pair[0], 'content_settings': content_settings, 'metadata': metadata, 'max_connections': max_connections, } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): create_file_args['validate_content'] = validate_content client.create_file_from_path(**create_file_args) return client.make_file_url(destination, dir_name, file_name) return list(_upload_action(f) for f in source_files)
def storage_file_download_batch(client, source, destination, pattern=None, dryrun=False, validate_content=False, max_connections=1): """ Download files from file share to local directory in batch """ from .util import glob_files_remotely, mkdir_p source_files = glob_files_remotely(client, source, pattern) if dryrun: source_files_list = list(source_files) logger = get_az_logger(__name__) logger.warning('upload files to file share') logger.warning(' account %s', client.account_name) logger.warning(' share %s', source) logger.warning('destination %s', destination) logger.warning(' pattern %s', pattern) logger.warning(' total %d', len(source_files_list)) logger.warning(' operations') for f in source_files_list: logger.warning(' - %s/%s => %s', f[0], f[1], os.path.join(destination, *f)) return [] def _download_action(pair): destination_dir = os.path.join(destination, pair[0]) mkdir_p(destination_dir) get_file_args = { 'share_name': source, 'directory_name': pair[0], 'file_name': pair[1], 'file_path': os.path.join(destination, *pair), 'max_connections': max_connections } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): get_file_args['validate_content'] = validate_content client.get_file_to_path(**get_file_args) return client.make_file_url(source, *pair) return list(_download_action(f) for f in source_files)
def storage_file_upload_batch(client, destination, source, pattern=None, dryrun=False, validate_content=False, content_settings=None, max_connections=1, metadata=None): """ Upload local files to Azure Storage File Share in batch """ from .util import glob_files_locally source_files = [c for c in glob_files_locally(source, pattern)] logger = get_az_logger(__name__) settings_class = get_sdk(ResourceType.DATA_STORAGE, 'file.models#ContentSettings') if dryrun: logger.info('upload files to file share') logger.info(' account %s', client.account_name) logger.info(' share %s', destination) logger.info(' total %d', len(source_files or [])) return [{'File': client.make_file_url(destination, os.path.dirname(src), os.path.basename(dst)), 'Type': guess_content_type(src, content_settings, settings_class).content_type} for src, dst in source_files] # TODO: Performance improvement # 1. Upload files in parallel def _upload_action(src, dst): dir_name = os.path.dirname(dst) file_name = os.path.basename(dst) _make_directory_in_files_share(client, destination, dir_name) create_file_args = { 'share_name': destination, 'directory_name': dir_name, 'file_name': file_name, 'local_file_path': src, 'content_settings': guess_content_type(src, content_settings, settings_class), 'metadata': metadata, 'max_connections': max_connections, } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): create_file_args['validate_content'] = validate_content logger.warning('uploading %s', src) client.create_file_from_path(**create_file_args) return client.make_file_url(destination, dir_name, file_name) return list(_upload_action(src, dst) for src, dst in source_files)
def __call__(self, poller): from msrest.exceptions import ClientException correlation_message = '' self.progress_controller.begin() correlation_id = None az_logger = azlogging.get_az_logger() is_verbose = any(handler.level <= logs.INFO for handler in az_logger.handlers) while not poller.done(): self.progress_controller.add(message='Running') try: # pylint: disable=protected-access correlation_id = json.loads( poller._response.__dict__['_content'].decode())['properties']['correlationId'] correlation_message = 'Correlation ID: {}'.format(correlation_id) except: # pylint: disable=bare-except pass current_time = datetime.datetime.now() if is_verbose and current_time - self.last_progress_report >= datetime.timedelta(seconds=10): self.last_progress_report = current_time try: self._generate_template_progress(correlation_id) except Exception as ex: # pylint: disable=broad-except logger.warning('%s during progress reporting: %s', getattr(type(ex), '__name__', type(ex)), ex) try: self._delay() except KeyboardInterrupt: self.progress_controller.stop() logger.error('Long running operation wait cancelled. %s', correlation_message) raise try: result = poller.result() except ClientException as client_exception: from azure.cli.core.commands.arm import handle_long_running_operation_exception self.progress_controller.stop() handle_long_running_operation_exception(client_exception) self.progress_controller.end() return result
def storage_file_upload_batch(client, destination, source, pattern=None, dryrun=False, validate_content=False, content_settings=None, max_connections=1, metadata=None): """ Upload local files to Azure Storage File Share in batch """ from .util import glob_files_locally source_files = [c for c in glob_files_locally(source, pattern)] if dryrun: logger = get_az_logger(__name__) logger.warning('upload files to file share') logger.warning(' account %s', client.account_name) logger.warning(' share %s', destination) logger.warning(' total %d', len(source_files or [])) logger.warning(' operations') for f in source_files or []: logger.warning(' - %s => %s', *f) return [] # TODO: Performance improvement # 1. Upload files in parallel def _upload_action(source_pair): dir_name = os.path.dirname(source_pair[1]) file_name = os.path.basename(source_pair[1]) _make_directory_in_files_share(client, destination, dir_name) client.create_file_from_path(share_name=destination, directory_name=dir_name, file_name=file_name, local_file_path=source_pair[0], content_settings=content_settings, metadata=metadata, max_connections=max_connections, validate_content=validate_content) return client.make_file_url(destination, dir_name, file_name) return list(_upload_action(f) for f in source_files)
def storage_file_download_batch(client, source, destination, pattern=None, dryrun=False, validate_content=False, max_connections=1): """ Download files from file share to local directory in batch """ from .util import glob_files_remotely, mkdir_p source_files = glob_files_remotely(client, source, pattern) if dryrun: source_files_list = list(source_files) logger = get_az_logger(__name__) logger.warning('upload files to file share') logger.warning(' account %s', client.account_name) logger.warning(' share %s', source) logger.warning('destination %s', destination) logger.warning(' pattern %s', pattern) logger.warning(' total %d', len(source_files_list)) logger.warning(' operations') for f in source_files_list: logger.warning(' - %s/%s => %s', f[0], f[1], os.path.join(destination, *f)) return [] def _download_action(pair): destination_dir = os.path.join(destination, pair[0]) mkdir_p(destination_dir) client.get_file_to_path(source, directory_name=pair[0], file_name=pair[1], file_path=os.path.join(destination, *pair), validate_content=validate_content, max_connections=max_connections) return client.make_file_url(source, *pair) return list(_download_action(f) for f in source_files)
def storage_blob_copy_batch(client, source_client, destination_container=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False): """Copy a group of blob or files to a blob container.""" logger = None if dryrun: logger = get_az_logger(__name__) logger.warning('copy files or blobs to blob container') logger.warning(' account %s', client.account_name) logger.warning(' container %s', destination_container) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') if source_container: # copy blobs for blob container # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client( client) if not source_sas and client.account_name != source_client.account_name: # when the blob is copied across storage account without sas, generate a short lived # sas for it source_sas = create_short_lived_container_sas( source_client.account_name, source_client.account_key, source_container) def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _copy_blob_to_blob_container(client, source_client, destination_container, source_container, source_sas, blob_name) return list( filter_none( action_blob_copy(blob) for blob in collect_blobs( source_client, source_container, pattern))) elif source_share: # copy blob from file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_file_share_from_storage_client( client) if not source_sas and client.account_name != source_client.account_name: # when the file is copied across storage account without sas, generate a short lived sas source_sas = create_short_lived_share_sas( source_client.account_name, source_client.account_key, source_share) def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _copy_file_to_blob_container(client, source_client, destination_container, source_share, source_sas, dir_name, file_name) return list( filter_none( action_file_copy(file) for file in collect_files( source_client, source_share, pattern))) else: raise ValueError( 'Fail to find source. Neither blob container or file share is specified' )
def storage_blob_upload_batch(client, source, destination, pattern=None, source_files=None, destination_container_name=None, blob_type=None, content_settings=None, metadata=None, validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, dryrun=False): """ Upload files to storage container as blobs :param str source: The directory where the files to be uploaded. :param str destination: The string represents the destination of this upload operation. The source can be the container URL or the container name. When the source is the container URL, the storage account name will parsed from the URL. :param str pattern: The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]', and '[!seq]'. :param bool dryrun: Show the summary of the operations to be taken instead of actually upload the file(s) :param string if_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag matches the value specified. :param string if_none_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag does not match the value specified. Specify the wildcard character (*) to perform the operation only if the resource does not exist, and fail the operation if it does exist. """ def _append_blob(file_path, blob_name): if not client.exists(destination_container_name, blob_name): client.create_blob(container_name=destination_container_name, blob_name=blob_name, content_settings=content_settings, metadata=metadata, lease_id=lease_id, if_modified_since=if_modified_since, if_match=if_match, if_none_match=if_none_match, timeout=timeout) return client.append_blob_from_path( container_name=destination_container_name, blob_name=blob_name, file_path=file_path, progress_callback=lambda c, t: None, validate_content=validate_content, maxsize_condition=maxsize_condition, lease_id=lease_id, timeout=timeout) def _upload_blob(file_path, blob_name): return client.create_blob_from_path( container_name=destination_container_name, blob_name=blob_name, file_path=file_path, progress_callback=lambda c, t: None, content_settings=content_settings, metadata=metadata, validate_content=validate_content, max_connections=max_connections, lease_id=lease_id, if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, if_match=if_match, if_none_match=if_none_match, timeout=timeout) upload_action = _upload_blob if blob_type == 'block' or blob_type == 'page' else _append_blob if dryrun: logger = get_az_logger(__name__) logger.warning('upload action: from %s to %s', source, destination) logger.warning(' pattern %s', pattern) logger.warning(' container %s', destination_container_name) logger.warning(' type %s', blob_type) logger.warning(' total %d', len(source_files)) logger.warning(' operations') for f in source_files or []: logger.warning(' - %s => %s', *f) else: for f in source_files or []: print('uploading {}'.format(f[0])) upload_action(*f)
def storage_file_copy_batch(client, source_client, destination_share=None, destination_path=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False, metadata=None, timeout=None): """ Copy a group of files asynchronously """ logger = None if dryrun: logger = get_az_logger(__name__) logger.warning('copy files or blobs to file share') logger.warning(' account %s', client.account_name) logger.warning(' share %s', destination_share) logger.warning(' path %s', destination_path) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') if source_container: # copy blobs to file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client(client) # the cache of existing directories in the destination file share. the cache helps to avoid # repeatedly create existing directory so as to optimize the performance. existing_dirs = set([]) if not source_sas and client.account_name != source_client.account_name: # when blob is copied across storage account without sas, generate a short lived # sas for it source_sas = create_short_lived_container_sas(source_client.account_name, source_client.account_key, source_container) def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _create_file_and_directory_from_blob( client, source_client, destination_share, source_container, source_sas, blob_name, destination_dir=destination_path, metadata=metadata, timeout=timeout, existing_dirs=existing_dirs) return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, source_container, pattern))) elif source_share: # copy files from share to share # if the source client is None, assume the file share is in the same storage account as # destination, therefore client is reused. source_client = source_client or client # the cache of existing directories in the destination file share. the cache helps to avoid # repeatedly create existing directory so as to optimize the performance. existing_dirs = set([]) if not source_sas and client.account_name != source_client.account_name: # when file is copied across storage account without sas, generate a short lived # sas for it source_sas = create_short_lived_share_sas(source_client.account_name, source_client.account_key, source_share) def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _create_file_and_directory_from_file( client, source_client, destination_share, source_share, source_sas, dir_name, file_name, destination_dir=destination_path, metadata=metadata, timeout=timeout, existing_dirs=existing_dirs) return list(filter_none(action_file_copy(file) for file in collect_files(source_client, source_share, pattern))) else: # won't happen, the validator should ensure either source_container or source_share is set raise ValueError('Fail to find source. Neither blob container or file share is specified.')
import os import random import socket import string import sys import threading from subprocess import PIPE, CalledProcessError, Popen, check_output from time import sleep import azure.cli.command_modules.project.naming as naming import azure.cli.command_modules.project.settings as settings import azure.cli.core.azlogging as azlogging # pylint: disable=invalid-name from azure.cli.core._util import CLIError logger = azlogging.get_az_logger(__name__) # pylint: disable=invalid-name def get_random_registry_name(): """ Gets a random name for the Azure Container Registry """ return get_random_string(only_letters=True) def get_random_string(length=6, only_letters=False): """ Gets a random lowercase string made from ascii letters and digits """
def test_get_az_logger(self): az_logger = azlogging.get_az_logger() self.assertEqual(az_logger.name, 'az')
def storage_file_upload_batch(client, destination, source, pattern=None, dryrun=False, validate_content=False, content_settings=None, max_connections=1, metadata=None): """ Upload local files to Azure Storage File Share in batch """ from .util import glob_files_locally source_files = [c for c in glob_files_locally(source, pattern)] logger = get_az_logger(__name__) settings_class = get_sdk(ResourceType.DATA_STORAGE, 'file.models#ContentSettings') if dryrun: logger.info('upload files to file share') logger.info(' account %s', client.account_name) logger.info(' share %s', destination) logger.info(' total %d', len(source_files or [])) return [{ 'File': client.make_file_url(destination, os.path.dirname(src), os.path.basename(dst)), 'Type': guess_content_type(src, content_settings, settings_class).content_type } for src, dst in source_files] # TODO: Performance improvement # 1. Upload files in parallel def _upload_action(src, dst): dir_name = os.path.dirname(dst) file_name = os.path.basename(dst) _make_directory_in_files_share(client, destination, dir_name) create_file_args = { 'share_name': destination, 'directory_name': dir_name, 'file_name': file_name, 'local_file_path': src, 'content_settings': guess_content_type(src, content_settings, settings_class), 'metadata': metadata, 'max_connections': max_connections, } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): create_file_args['validate_content'] = validate_content logger.warning('uploading %s', src) client.create_file_from_path(**create_file_args) return client.make_file_url(destination, dir_name, file_name) return list(_upload_action(src, dst) for src, dst in source_files)
def validator(namespace): type_field_name = '{}_type'.format(property_name) property_val = getattr(namespace, property_name, None) parent_val = getattr(namespace, parent_name, None) if parent_name else None # Check for the different scenarios (order matters) # 1) provided value indicates None (pair of empty quotes) if property_val in ('', '""', "''") or (property_val is None and default_none): if not allow_none: raise CLIError('{} cannot be None.'.format(property_option)) setattr(namespace, type_field_name, 'none') setattr(namespace, property_name, None) if parent_name and parent_val: logger = azlogging.get_az_logger(__name__) logger.warning('Ignoring: %s %s', parent_option, parent_val) setattr(namespace, parent_name, None) return # SUCCESS # Create a resource ID we can check for existence. (resource_id_parts, value_was_id) = _validate_name_or_id(namespace.resource_group_name, property_val, property_type, parent_val, parent_type) # 2) resource exists if resource_exists(**resource_id_parts): setattr(namespace, type_field_name, 'existingId') setattr(namespace, property_name, resource_id(**resource_id_parts)) if parent_val: if value_was_id: logger = azlogging.get_az_logger(__name__) logger.warning('Ignoring: %s %s', parent_option, parent_val) setattr(namespace, parent_name, None) return # SUCCESS # if a parent name was required but not specified, raise a usage error if has_parent and not value_was_id and not parent_val and not allow_new: raise ValueError( 'incorrect usage: {0} ID | {0} NAME {1} NAME'.format( property_option, parent_option)) # if non-existent ID was supplied, throw error depending on whether a new resource can # be created. if value_was_id: usage_message = '{} NAME'.format(property_option) if not has_parent \ else '{} NAME [{} NAME]'.format(property_option, parent_option) action_message = 'Specify ( {} ) to create a new resource.'.format(usage_message) if \ allow_new else 'Create the required resource and try again.' raise CLIError('{} {} does not exist. {}'.format( property_name, property_val, action_message)) # 3) try to create new resource if allow_new: setattr(namespace, type_field_name, 'new') else: raise CLIError( '{} {} does not exist. Create the required resource and try again.' .format(property_name, property_val))
def storage_blob_upload_batch( client, source, destination, pattern=None, source_files=None, # pylint: disable=too-many-locals destination_container_name=None, blob_type=None, content_settings=None, metadata=None, validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, dryrun=False): """ Upload files to storage container as blobs :param str source: The directory where the files to be uploaded. :param str destination: The string represents the destination of this upload operation. The source can be the container URL or the container name. When the source is the container URL, the storage account name will parsed from the URL. :param str pattern: The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]', and '[!seq]'. :param bool dryrun: Show the summary of the operations to be taken instead of actually upload the file(s) :param string if_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag matches the value specified. :param string if_none_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag does not match the value specified. Specify the wildcard character (*) to perform the operation only if the resource does not exist, and fail the operation if it does exist. """ def _append_blob(file_path, blob_name, blob_content_settings): if not client.exists(destination_container_name, blob_name): client.create_blob(container_name=destination_container_name, blob_name=blob_name, content_settings=blob_content_settings, metadata=metadata, lease_id=lease_id, if_modified_since=if_modified_since, if_match=if_match, if_none_match=if_none_match, timeout=timeout) append_blob_args = { 'container_name': destination_container_name, 'blob_name': blob_name, 'file_path': file_path, 'progress_callback': lambda c, t: None, 'maxsize_condition': maxsize_condition, 'lease_id': lease_id, 'timeout': timeout } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): append_blob_args['validate_content'] = validate_content return client.append_blob_from_path(**append_blob_args) def _upload_blob(file_path, blob_name, blob_content_settings): create_blob_args = { 'container_name': destination_container_name, 'blob_name': blob_name, 'file_path': file_path, 'progress_callback': lambda c, t: None, 'content_settings': blob_content_settings, 'metadata': metadata, 'max_connections': max_connections, 'lease_id': lease_id, 'if_modified_since': if_modified_since, 'if_unmodified_since': if_unmodified_since, 'if_match': if_match, 'if_none_match': if_none_match, 'timeout': timeout } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): create_blob_args['validate_content'] = validate_content return client.create_blob_from_path(**create_blob_args) def _create_return_result(blob_name, blob_content_settings, upload_result=None): return { 'Blob': client.make_blob_url(destination_container_name, blob_name), 'Type': blob_content_settings.content_type, 'Last Modified': upload_result.last_modified if upload_result else None, 'eTag': upload_result.etag if upload_result else None } upload_action = _upload_blob if blob_type == 'block' or blob_type == 'page' else _append_blob logger = get_az_logger(__name__) settings_class = get_sdk(ResourceType.DATA_STORAGE, 'blob.models#ContentSettings') results = [] if dryrun: logger.info('upload action: from %s to %s', source, destination) logger.info(' pattern %s', pattern) logger.info(' container %s', destination_container_name) logger.info(' type %s', blob_type) logger.info(' total %d', len(source_files)) results = [] for src, dst in source_files or []: results.append( _create_return_result( dst, guess_content_type(src, content_settings, settings_class))) else: for src, dst in source_files or []: logger.warning('uploading {}'.format(src)) guessed_content_settings = guess_content_type( src, content_settings, settings_class) results.append( _create_return_result( dst, guessed_content_settings, upload_action(src, dst, guessed_content_settings))) return results
def test_get_az_logger_module(self): az_module_logger = azlogging.get_az_logger('azure.cli.module') self.assertEqual(az_module_logger.name, 'az.azure.cli.module')
# -------------------------------------------------------------------------------------------- # AZURE CLI RBAC TEST DEFINITIONS import json import os import tempfile import time import unittest from azure.cli.testsdk import (LiveScenarioTest, ScenarioTest, ResourceGroupPreparer, KeyVaultPreparer, JMESPathCheck as JMESPathCheckV2) import azure.cli.core.azlogging as azlogging from azure.cli.testsdk.vcr_test_base import (VCRTestBase, JMESPathCheck, ResourceGroupVCRTestBase, NoneCheck, MOCKED_SUBSCRIPTION_ID) logger = azlogging.get_az_logger(__name__) class RbacSPSecretScenarioTest(LiveScenarioTest): @ResourceGroupPreparer(name_prefix='cli_create_rbac_sp_minimal') def test_create_for_rbac_with_secret(self, resource_group): sp_name = 'http://{}'.format(resource_group) try: self.cmd('ad sp create-for-rbac -n {}2'.format(sp_name), checks=[ JMESPathCheckV2('name', sp_name) ]) finally: self.cmd('ad app delete --id {}2'.format(sp_name)) @ResourceGroupPreparer(name_prefix='cli_create_rbac_sp_with_password')
def _log_hostname(): import socket import azure.cli.core.azlogging as azlogging logger = azlogging.get_az_logger(__name__) logger.warning("A Cloud Shell credential problem occurred. When you report the issue with the error " "below, please mention the hostname '%s'", socket.gethostname())
def storage_blob_upload_batch( client, source, destination, pattern=None, source_files=None, # pylint: disable=too-many-locals destination_container_name=None, blob_type=None, content_settings=None, metadata=None, validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, dryrun=False): def _create_return_result(blob_name, blob_content_settings, upload_result=None): return { 'Blob': client.make_blob_url(destination_container_name, blob_name), 'Type': blob_content_settings.content_type, 'Last Modified': upload_result.last_modified if upload_result else None, 'eTag': upload_result.etag if upload_result else None } logger = get_az_logger(__name__) settings_class = get_sdk(ResourceType.DATA_STORAGE, 'blob.models#ContentSettings') results = [] if dryrun: logger.info('upload action: from %s to %s', source, destination) logger.info(' pattern %s', pattern) logger.info(' container %s', destination_container_name) logger.info(' type %s', blob_type) logger.info(' total %d', len(source_files)) results = [] for src, dst in source_files or []: results.append( _create_return_result( dst, guess_content_type(src, content_settings, settings_class))) else: for src, dst in source_files or []: logger.warning('uploading {}'.format(src)) guessed_content_settings = guess_content_type( src, content_settings, settings_class) result = upload_blob(client, destination_container_name, dst, src, blob_type=blob_type, content_settings=guessed_content_settings, metadata=metadata, validate_content=validate_content, maxsize_condition=maxsize_condition, max_connections=max_connections, lease_id=lease_id, if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, if_match=if_match, if_none_match=if_none_match, timeout=timeout) results.append( _create_return_result(dst, guessed_content_settings, result)) return results
def storage_blob_upload_batch(client, source, destination, pattern=None, source_files=None, destination_container_name=None, blob_type=None, content_settings=None, metadata=None, validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, dryrun=False): """ Upload files to storage container as blobs :param str source: The directory where the files to be uploaded. :param str destination: The string represents the destination of this upload operation. The source can be the container URL or the container name. When the source is the container URL, the storage account name will parsed from the URL. :param str pattern: The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]', and '[!seq]'. :param bool dryrun: Show the summary of the operations to be taken instead of actually upload the file(s) :param string if_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag matches the value specified. :param string if_none_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag does not match the value specified. Specify the wildcard character (*) to perform the operation only if the resource does not exist, and fail the operation if it does exist. """ def _append_blob(file_path, blob_name): if not client.exists(destination_container_name, blob_name): client.create_blob( container_name=destination_container_name, blob_name=blob_name, content_settings=content_settings, metadata=metadata, lease_id=lease_id, if_modified_since=if_modified_since, if_match=if_match, if_none_match=if_none_match, timeout=timeout) return client.append_blob_from_path( container_name=destination_container_name, blob_name=blob_name, file_path=file_path, progress_callback=lambda c, t: None, validate_content=validate_content, maxsize_condition=maxsize_condition, lease_id=lease_id, timeout=timeout) def _upload_blob(file_path, blob_name): return client.create_blob_from_path( container_name=destination_container_name, blob_name=blob_name, file_path=file_path, progress_callback=lambda c, t: None, content_settings=content_settings, metadata=metadata, validate_content=validate_content, max_connections=max_connections, lease_id=lease_id, if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, if_match=if_match, if_none_match=if_none_match, timeout=timeout) upload_action = _upload_blob if blob_type == 'block' or blob_type == 'page' else _append_blob if dryrun: logger = get_az_logger(__name__) logger.warning('upload action: from %s to %s', source, destination) logger.warning(' pattern %s', pattern) logger.warning(' container %s', destination_container_name) logger.warning(' type %s', blob_type) logger.warning(' total %d', len(source_files)) logger.warning(' operations') for f in source_files or []: logger.warning(' - %s => %s', *f) else: for f in source_files or []: print('uploading {}'.format(f[0])) upload_action(*f)
def storage_blob_copy_batch(client, source_client, destination_container=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False): """Copy a group of blob or files to a blob container.""" logger = None if dryrun: logger = get_az_logger(__name__) logger.warning('copy files or blobs to blob container') logger.warning(' account %s', client.account_name) logger.warning(' container %s', destination_container) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') if source_container: # copy blobs for blob container # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client(client) if not source_sas and client.account_name != source_client.account_name: # when the blob is copied across storage account without sas, generate a short lived # sas for it source_sas = create_short_lived_container_sas(source_client.account_name, source_client.account_key, source_container) def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _copy_blob_to_blob_container(client, source_client, destination_container, source_container, source_sas, blob_name) return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, source_container, pattern))) elif source_share: # copy blob from file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_file_share_from_storage_client(client) if not source_sas and client.account_name != source_client.account_name: # when the file is copied across storage account without sas, generate a short lived sas source_sas = create_short_lived_share_sas(source_client.account_name, source_client.account_key, source_share) def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _copy_file_to_blob_container(client, source_client, destination_container, source_share, source_sas, dir_name, file_name) return list(filter_none(action_file_copy(file) for file in collect_files(source_client, source_share, pattern))) else: raise ValueError('Fail to find source. Neither blob container or file share is specified')
import sys import os import uuid import argparse from azure.cli.core.parser import AzCliCommandParser, enable_autocomplete from azure.cli.core._output import CommandResultItem import azure.cli.core.extensions import azure.cli.core._help as _help import azure.cli.core.azlogging as azlogging from azure.cli.core.util import todict, truncate_text, CLIError, read_file_content from azure.cli.core._config import az_config import azure.cli.core.commands.progress as progress import azure.cli.core.telemetry as telemetry logger = azlogging.get_az_logger(__name__) ARGCOMPLETE_ENV_NAME = '_ARGCOMPLETE' class Configuration(object): # pylint: disable=too-few-public-methods """The configuration object tracks session specific data such as output formats, available commands etc. """ def __init__(self): self.output_format = None def get_command_table(self, argv=None): # pylint: disable=no-self-use import azure.cli.core.commands as commands # Find the first noun on the command line and only load commands from that
def storage_blob_upload_batch(client, source, destination, pattern=None, source_files=None, # pylint: disable=too-many-locals destination_container_name=None, blob_type=None, content_settings=None, metadata=None, validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, dryrun=False): """ Upload files to storage container as blobs :param str source: The directory where the files to be uploaded. :param str destination: The string represents the destination of this upload operation. The source can be the container URL or the container name. When the source is the container URL, the storage account name will parsed from the URL. :param str pattern: The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]', and '[!seq]'. :param bool dryrun: Show the summary of the operations to be taken instead of actually upload the file(s) :param string if_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag matches the value specified. :param string if_none_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag does not match the value specified. Specify the wildcard character (*) to perform the operation only if the resource does not exist, and fail the operation if it does exist. """ def _append_blob(file_path, blob_name, blob_content_settings): if not client.exists(destination_container_name, blob_name): client.create_blob( container_name=destination_container_name, blob_name=blob_name, content_settings=blob_content_settings, metadata=metadata, lease_id=lease_id, if_modified_since=if_modified_since, if_match=if_match, if_none_match=if_none_match, timeout=timeout) append_blob_args = { 'container_name': destination_container_name, 'blob_name': blob_name, 'file_path': file_path, 'progress_callback': lambda c, t: None, 'maxsize_condition': maxsize_condition, 'lease_id': lease_id, 'timeout': timeout } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): append_blob_args['validate_content'] = validate_content return client.append_blob_from_path(**append_blob_args) def _upload_blob(file_path, blob_name, blob_content_settings): create_blob_args = { 'container_name': destination_container_name, 'blob_name': blob_name, 'file_path': file_path, 'progress_callback': lambda c, t: None, 'content_settings': blob_content_settings, 'metadata': metadata, 'max_connections': max_connections, 'lease_id': lease_id, 'if_modified_since': if_modified_since, 'if_unmodified_since': if_unmodified_since, 'if_match': if_match, 'if_none_match': if_none_match, 'timeout': timeout } if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'): create_blob_args['validate_content'] = validate_content return client.create_blob_from_path(**create_blob_args) def _create_return_result(blob_name, blob_content_settings, upload_result=None): return { 'Blob': client.make_blob_url(destination_container_name, blob_name), 'Type': blob_content_settings.content_type, 'Last Modified': upload_result.last_modified if upload_result else None, 'eTag': upload_result.etag if upload_result else None} upload_action = _upload_blob if blob_type == 'block' or blob_type == 'page' else _append_blob logger = get_az_logger(__name__) settings_class = get_sdk(ResourceType.DATA_STORAGE, 'blob.models#ContentSettings') results = [] if dryrun: logger.info('upload action: from %s to %s', source, destination) logger.info(' pattern %s', pattern) logger.info(' container %s', destination_container_name) logger.info(' type %s', blob_type) logger.info(' total %d', len(source_files)) results = [] for src, dst in source_files or []: results.append(_create_return_result(dst, guess_content_type(src, content_settings, settings_class))) else: for src, dst in source_files or []: logger.warning('uploading {}'.format(src)) guessed_content_settings = guess_content_type(src, content_settings, settings_class) results.append( _create_return_result(dst, guessed_content_settings, upload_action(src, dst, guessed_content_settings))) return results