def process_blob_upload_batch_parameters(cmd, namespace):
"""Process the source and destination of storage blob upload command"""
import os
# 1. quick check
if not os.path.exists(namespace.source):
raise ValueError('incorrect usage: source {} does not exist'.format(namespace.source))
if not os.path.isdir(namespace.source):
raise ValueError('incorrect usage: source must be a directory')
# 2. try to extract account name and container name from destination string
from .storage_url_helpers import StorageResourceIdentifier
identifier = StorageResourceIdentifier(cmd.cli_ctx.cloud, namespace.destination)
if not identifier.is_url():
namespace.destination_container_name = namespace.destination
elif identifier.blob is not None:
raise ValueError('incorrect usage: destination cannot be a blob url')
else:
namespace.destination_container_name = identifier.container
if namespace.account_name:
if namespace.account_name != identifier.account_name:
raise ValueError(
'The given storage account name is not consistent with the account name in the destination URI')
else:
namespace.account_name = identifier.account_name
if not (namespace.account_key or namespace.sas_token or namespace.connection_string):
validate_client_parameters(cmd, namespace)
# it is possible the account name be overwritten by the connection string
if namespace.account_name != identifier.account_name:
raise ValueError(
'The given storage account name is not consistent with the account name in the destination URI')
if not (namespace.account_key or namespace.sas_token or namespace.connection_string):
raise ValueError('Missing storage account credential information.')
# 3. collect the files to be uploaded
namespace.source = os.path.realpath(namespace.source)
namespace.source_files = [c for c in glob_files_locally(namespace.source, namespace.pattern)]
# 4. determine blob type
if namespace.blob_type is None:
vhd_files = [f for f in namespace.source_files if f[0].endswith('.vhd')]
if any(vhd_files) and len(vhd_files) == len(namespace.source_files):
# when all the listed files are vhd files use page
namespace.blob_type = 'page'
elif any(vhd_files):
# source files contain vhd files but not all of them
from knack.util import CLIError
raise CLIError("""Fail to guess the required blob type. Type of the files to be
uploaded are not consistent. Default blob type for .vhd files is "page", while
others are "block". You can solve this problem by either explicitly set the blob
type or ensure the pattern matches a correct set of files.""")
else:
namespace.blob_type = 'block'
def process_blob_upload_batch_parameters(cmd, namespace):
"""Process the source and destination of storage blob upload command"""
import os
# 1. quick check
if not os.path.exists(namespace.source):
raise ValueError('incorrect usage: source {} does not exist'.format(namespace.source))
if not os.path.isdir(namespace.source):
raise ValueError('incorrect usage: source must be a directory')
# 2. try to extract account name and container name from destination string
from .storage_url_helpers import StorageResourceIdentifier
identifier = StorageResourceIdentifier(cmd.cli_ctx.cloud, namespace.destination)
if not identifier.is_url():
namespace.destination_container_name = namespace.destination
elif identifier.blob is not None:
raise ValueError('incorrect usage: destination cannot be a blob url')
else:
namespace.destination_container_name = identifier.container
if namespace.account_name:
if namespace.account_name != identifier.account_name:
raise ValueError(
'The given storage account name is not consistent with the account name in the destination URI')
else:
namespace.account_name = identifier.account_name
if not (namespace.account_key or namespace.sas_token or namespace.connection_string):
validate_client_parameters(cmd, namespace)
# it is possible the account name be overwritten by the connection string
if namespace.account_name != identifier.account_name:
raise ValueError(
'The given storage account name is not consistent with the account name in the destination URI')
if not (namespace.account_key or namespace.sas_token or namespace.connection_string):
raise ValueError('Missing storage account credential information.')
# 3. collect the files to be uploaded
namespace.source = os.path.realpath(namespace.source)
namespace.source_files = [c for c in glob_files_locally(namespace.source, namespace.pattern)]
# 4. determine blob type
if namespace.blob_type is None:
vhd_files = [f for f in namespace.source_files if f[0].endswith('.vhd')]
if any(vhd_files) and len(vhd_files) == len(namespace.source_files):
# when all the listed files are vhd files use page
namespace.blob_type = 'page'
elif any(vhd_files):
# source files contain vhd files but not all of them
from knack.util import CLIError
raise CLIError("""Fail to guess the required blob type. Type of the files to be
uploaded are not consistent. Default blob type for .vhd files is "page", while
others are "block". You can solve this problem by either explicitly set the blob
type or ensure the pattern matches a correct set of files.""")
else:
namespace.blob_type = 'block'
def storage_file_upload_batch(cmd, client, destination, source, destination_path=None, pattern=None, dryrun=False,
validate_content=False, content_settings=None, max_connections=1, metadata=None,
progress_callback=None):
""" Upload local files to Azure Storage File Share in batch """
from azure.cli.command_modules.storage.util import glob_files_locally, normalize_blob_file_path
source_files = [c for c in glob_files_locally(source, pattern)]
logger = get_logger(__name__)
settings_class = cmd.get_models('file.models#ContentSettings')
if dryrun:
logger.info('upload files to file share')
logger.info(' account %s', client.account_name)
logger.info(' share %s', destination)
logger.info(' total %d', len(source_files))
return [{'File': client.make_file_url(destination, os.path.dirname(dst) or None, os.path.basename(dst)),
'Type': guess_content_type(src, content_settings, settings_class).content_type} for src, dst in
source_files]
# TODO: Performance improvement
# 1. Upload files in parallel
def _upload_action(src, dst):
dst = normalize_blob_file_path(destination_path, dst)
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {'share_name': destination, 'directory_name': dir_name, 'file_name': file_name,
'local_file_path': src, 'progress_callback': progress_callback,
'content_settings': guess_content_type(src, content_settings, settings_class),
'metadata': metadata, 'max_connections': max_connections}
if cmd.supported_api_version(min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
return list(_upload_action(src, dst) for src, dst in source_files)
def process_blob_upload_batch_parameters(cmd, namespace):
"""Process the source and destination of storage blob upload command"""
import os
# 1. quick check
if not os.path.exists(namespace.source) or not os.path.isdir(namespace.source):
raise ValueError('incorrect usage: source must be an existing directory')
# 2. try to extract account name and container name from destination string
_process_blob_batch_container_parameters(cmd, namespace, source=False)
# 3. collect the files to be uploaded
namespace.source = os.path.realpath(namespace.source)
namespace.source_files = [c for c in glob_files_locally(namespace.source, namespace.pattern)]
# 4. determine blob type
if namespace.blob_type is None:
vhd_files = [f for f in namespace.source_files if f[0].endswith('.vhd')]
if any(vhd_files) and len(vhd_files) == len(namespace.source_files):
# when all the listed files are vhd files use page
namespace.blob_type = 'page'
elif any(vhd_files):
# source files contain vhd files but not all of them
from knack.util import CLIError
raise CLIError("""Fail to guess the required blob type. Type of the files to be
uploaded are not consistent. Default blob type for .vhd files is "page", while
others are "block". You can solve this problem by either explicitly set the blob
type or ensure the pattern matches a correct set of files.""")
else:
namespace.blob_type = 'block'
# 5. call other validators
validate_metadata(namespace)
t_blob_content_settings = cmd.loader.get_sdk('blob.models#ContentSettings')
get_content_setting_validator(t_blob_content_settings, update=False)(cmd, namespace)
add_progress_callback(cmd, namespace)
