def storage_blob_delete_batch(client, source, source_container_name, pattern=None, lease_id=None,
delete_snapshots=None, if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None, dryrun=False):
@check_precondition_success
def _delete_blob(blob_name):
delete_blob_args = {
'container_name': source_container_name,
'blob_name': blob_name,
'lease_id': lease_id,
'delete_snapshots': delete_snapshots,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
return client.delete_blob(**delete_blob_args)
logger = get_logger(__name__)
source_blobs = list(collect_blobs(client, source_container_name, pattern))
if dryrun:
logger.warning('delete action: from %s', source)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for blob in source_blobs:
logger.warning(' - %s', blob)
return []
results = [result for include, result in (_delete_blob(blob) for blob in source_blobs) if include]
num_failures = len(source_blobs) - len(results)
if num_failures:
logger.warning('%s of %s blobs not deleted due to "Failed Precondition"', num_failures, len(source_blobs))
def storage_blob_copy_batch(cmd, client, source_client, container_name=None,
destination_path=None, source_container=None, source_share=None,
source_sas=None, pattern=None, dryrun=False):
"""Copy a group of blob or files to a blob container."""
if dryrun:
logger.warning('copy files or blobs to blob container')
logger.warning(' account %s', client.account_name)
logger.warning(' container %s', container_name)
logger.warning(' source %s', source_container or source_share)
logger.warning('source type %s', 'blob' if source_container else 'file')
logger.warning(' pattern %s', pattern)
logger.warning(' operations')
source_sas = source_sas.lstrip('?') if source_sas else source_sas
if source_container:
# copy blobs for blob container
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_blob_service_from_storage_client(cmd, client)
if not source_sas:
source_sas = create_short_lived_container_sas(cmd, source_client.account_name, source_client.account_key,
source_container)
# pylint: disable=inconsistent-return-statements
def action_blob_copy(blob_name):
if dryrun:
logger.warning(' - copy blob %s', blob_name)
else:
return _copy_blob_to_blob_container(client, source_client, container_name, destination_path,
source_container, source_sas, blob_name)
return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client,
source_container,
pattern)))
if source_share:
# copy blob from file share
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_file_share_from_storage_client(cmd, client)
if not source_sas:
source_sas = create_short_lived_share_sas(cmd, source_client.account_name, source_client.account_key,
source_share)
# pylint: disable=inconsistent-return-statements
def action_file_copy(file_info):
dir_name, file_name = file_info
if dryrun:
logger.warning(' - copy file %s', os.path.join(dir_name, file_name))
else:
return _copy_file_to_blob_container(client, source_client, container_name, destination_path,
source_share, source_sas, dir_name, file_name)
return list(filter_none(action_file_copy(file) for file in collect_files(cmd,
source_client,
source_share,
pattern)))
raise ValueError('Fail to find source. Neither blob container or file share is specified')
def storage_blob_delete_batch(client, source, source_container_name, pattern=None, lease_id=None,
delete_snapshots=None, if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None, dryrun=False):
@check_precondition_success
def _delete_blob(blob_name):
delete_blob_args = {
'container_name': source_container_name,
'blob_name': blob_name,
'lease_id': lease_id,
'delete_snapshots': delete_snapshots,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
return client.delete_blob(**delete_blob_args)
source_blobs = list(collect_blobs(client, source_container_name, pattern))
if dryrun:
logger = get_logger(__name__)
logger.warning('delete action: from %s', source)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for blob in source_blobs:
logger.warning(' - %s', blob)
return []
return [result for include, result in (_delete_blob(blob) for blob in source_blobs) if include]
def storage_blob_delete_batch(client, source, source_container_name, pattern=None, lease_id=None,
delete_snapshots=None, if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None, dryrun=False):
def _delete_blob(blob_name):
delete_blob_args = {
'container_name': source_container_name,
'blob_name': blob_name,
'lease_id': lease_id,
'delete_snapshots': delete_snapshots,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
response = client.delete_blob(**delete_blob_args)
return response
source_blobs = list(collect_blobs(client, source_container_name, pattern))
if dryrun:
logger = get_logger(__name__)
logger.warning('delete action: from %s', source)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for blob in source_blobs:
logger.warning(' - %s', blob)
return []
return [_delete_blob(blob) for blob in source_blobs]
def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False,
progress_callback=None, max_connections=2):
def _download_blob(blob_service, container, destination_folder, blob_name):
# TODO: try catch IO exception
destination_path = os.path.join(destination_folder, blob_name)
destination_folder = os.path.dirname(destination_path)
if not os.path.exists(destination_folder):
mkdir_p(destination_folder)
blob = blob_service.get_blob_to_path(container, blob_name, destination_path, max_connections=max_connections,
progress_callback=progress_callback)
return blob.name
source_blobs = list(collect_blobs(client, source_container_name, pattern))
if dryrun:
logger = get_logger(__name__)
logger.warning('download action: from %s to %s', source, destination)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for b in source_blobs:
logger.warning(' - %s', b)
return []
return list(_download_blob(client, source_container_name, destination, blob) for blob in source_blobs)
def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None,
dryrun=False):
"""
Download blobs in a container recursively
:param str source:
The string represents the source of this download operation. The source can be the
container URL or the container name. When the source is the container URL, the storage
account name will parsed from the URL.
:param str destination:
The string represents the destination folder of this download operation. The folder must
exist.
:param bool dryrun:
Show the summary of the operations to be taken instead of actually download the file(s)
:param str pattern:
The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]',
and '[!seq]'.
"""
source_blobs = list(collect_blobs(client, source_container_name, pattern))
if dryrun:
logger = get_az_logger(__name__)
logger.warning('download action: from %s to %s', source, destination)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for b in source_blobs or []:
logger.warning(' - %s', b)
return []
return list(_download_blob(client, source_container_name, destination, blob) for blob in source_blobs)
def storage_blob_download_batch(client,
source,
destination,
source_container_name,
pattern=None,
dryrun=False,
progress_callback=None,
max_connections=2):
def _download_blob(blob_service, container, destination_folder,
normalized_blob_name, blob_name):
# TODO: try catch IO exception
destination_path = os.path.join(destination_folder,
normalized_blob_name)
destination_folder = os.path.dirname(destination_path)
if not os.path.exists(destination_folder):
mkdir_p(destination_folder)
blob = blob_service.get_blob_to_path(
container,
blob_name,
destination_path,
max_connections=max_connections,
progress_callback=progress_callback)
return blob.name
source_blobs = collect_blobs(client, source_container_name, pattern)
blobs_to_download = {}
for blob_name in source_blobs:
# remove starting path seperator and normalize
normalized_blob_name = normalize_blob_file_path(None, blob_name)
if normalized_blob_name in blobs_to_download:
from knack.util import CLIError
raise CLIError(
'Multiple blobs with download path: `{}`. As a solution, use the `--pattern` parameter '
'to select for a subset of blobs to download OR utilize the `storage blob download` '
'command instead to download individual blobs.'.format(
normalized_blob_name))
blobs_to_download[normalized_blob_name] = blob_name
if dryrun:
logger = get_logger(__name__)
logger.warning('download action: from %s to %s', source, destination)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for b in source_blobs:
logger.warning(' - %s', b)
return []
return list(
_download_blob(client, source_container_name, destination, blob_normed,
blobs_to_download[blob_normed])
for blob_normed in blobs_to_download)
def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False,
progress_callback=None, max_connections=2):
def _download_blob(blob_service, container, destination_folder, normalized_blob_name, blob_name):
# TODO: try catch IO exception
destination_path = os.path.join(destination_folder, normalized_blob_name)
destination_folder = os.path.dirname(destination_path)
if not os.path.exists(destination_folder):
mkdir_p(destination_folder)
blob = blob_service.get_blob_to_path(container, blob_name, destination_path, max_connections=max_connections,
progress_callback=progress_callback)
return blob.name
source_blobs = collect_blobs(client, source_container_name, pattern)
blobs_to_download = {}
for blob_name in source_blobs:
# remove starting path seperator and normalize
normalized_blob_name = normalize_blob_file_path(None, blob_name)
if normalized_blob_name in blobs_to_download:
raise CLIError('Multiple blobs with download path: `{}`. As a solution, use the `--pattern` parameter '
'to select for a subset of blobs to download OR utilize the `storage blob download` '
'command instead to download individual blobs.'.format(normalized_blob_name))
blobs_to_download[normalized_blob_name] = blob_name
if dryrun:
logger = get_logger(__name__)
logger.warning('download action: from %s to %s', source, destination)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for b in source_blobs:
logger.warning(' - %s', b)
return []
# Tell progress reporter to reuse the same hook
if progress_callback:
progress_callback.reuse = True
results = []
for index, blob_normed in enumerate(blobs_to_download):
# add blob name and number to progress message
if progress_callback:
progress_callback.message = '{}/{}: "{}"'.format(
index + 1, len(blobs_to_download), blobs_to_download[blob_normed])
results.append(_download_blob(
client, source_container_name, destination, blob_normed, blobs_to_download[blob_normed]))
# end progress hook
if progress_callback:
progress_callback.hook.end()
return results
def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False):
source_blobs = list(collect_blobs(client, source_container_name, pattern))
if dryrun:
logger = get_logger(__name__)
logger.warning('download action: from %s to %s', source, destination)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for b in source_blobs:
logger.warning(' - %s', b)
return []
return list(_download_blob(client, source_container_name, destination, blob) for blob in source_blobs)
def storage_blob_download_batch(client,
source,
destination,
source_container_name,
pattern=None,
dryrun=False):
"""
Download blobs in a container recursively
:param str source:
The string represents the source of this download operation. The source can be the
container URL or the container name. When the source is the container URL, the storage
account name will parsed from the URL.
:param str destination:
The string represents the destination folder of this download operation. The folder must
exist.
:param bool dryrun:
Show the summary of the operations to be taken instead of actually download the file(s)
:param str pattern:
The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]',
and '[!seq]'.
"""
source_blobs = collect_blobs(client, source_container_name, pattern)
if dryrun:
logger = get_az_logger(__name__)
logger.warning('download action: from %s to %s', source, destination)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for b in source_blobs or []:
logger.warning(' - %s', b)
return []
else:
return list(
_download_blob(client, source_container_name, destination, blob)
for blob in source_blobs)
def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False,
progress_callback=None, max_connections=2):
def _download_blob(blob_service, container, destination_folder, normalized_blob_name, blob_name):
# TODO: try catch IO exception
destination_path = os.path.join(destination_folder, normalized_blob_name)
destination_folder = os.path.dirname(destination_path)
if not os.path.exists(destination_folder):
mkdir_p(destination_folder)
blob = blob_service.get_blob_to_path(container, blob_name, destination_path, max_connections=max_connections,
progress_callback=progress_callback)
return blob.name
source_blobs = collect_blobs(client, source_container_name, pattern)
blobs_to_download = {}
for blob_name in source_blobs:
# remove starting path seperator and normalize
normalized_blob_name = normalize_blob_file_path(None, blob_name)
if normalized_blob_name in blobs_to_download:
from knack.util import CLIError
raise CLIError('Multiple blobs with download path: `{}`. As a solution, use the `--pattern` parameter '
'to select for a subset of blobs to download OR utilize the `storage blob download` '
'command instead to download individual blobs.'.format(normalized_blob_name))
blobs_to_download[normalized_blob_name] = blob_name
if dryrun:
logger = get_logger(__name__)
logger.warning('download action: from %s to %s', source, destination)
logger.warning(' pattern %s', pattern)
logger.warning(' container %s', source_container_name)
logger.warning(' total %d', len(source_blobs))
logger.warning(' operations')
for b in source_blobs:
logger.warning(' - %s', b)
return []
return list(_download_blob(client, source_container_name, destination, blob_normed, blobs_to_download[blob_normed])
for blob_normed in blobs_to_download)
def storage_file_copy_batch(client, source_client,
destination_share=None, destination_path=None,
source_container=None, source_share=None, source_sas=None,
pattern=None, dryrun=False, metadata=None, timeout=None):
"""
Copy a group of files asynchronously
"""
logger = None
if dryrun:
logger = get_az_logger(__name__)
logger.warning('copy files or blobs to file share')
logger.warning(' account %s', client.account_name)
logger.warning(' share %s', destination_share)
logger.warning(' path %s', destination_path)
logger.warning(' source %s', source_container or source_share)
logger.warning('source type %s', 'blob' if source_container else 'file')
logger.warning(' pattern %s', pattern)
logger.warning(' operations')
if source_container:
# copy blobs to file share
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_blob_service_from_storage_client(client)
# the cache of existing directories in the destination file share. the cache helps to avoid
# repeatedly create existing directory so as to optimize the performance.
existing_dirs = set([])
if not source_sas and client.account_name != source_client.account_name:
# when blob is copied across storage account without sas, generate a short lived
# sas for it
source_sas = create_short_lived_container_sas(source_client.account_name,
source_client.account_key,
source_container)
def action_blob_copy(blob_name):
if dryrun:
logger.warning(' - copy blob %s', blob_name)
else:
return _create_file_and_directory_from_blob(
client, source_client, destination_share, source_container, source_sas,
blob_name, destination_dir=destination_path, metadata=metadata, timeout=timeout,
existing_dirs=existing_dirs)
return list(filter_none(action_blob_copy(blob) for blob in
collect_blobs(source_client, source_container, pattern)))
elif source_share:
# copy files from share to share
# if the source client is None, assume the file share is in the same storage account as
# destination, therefore client is reused.
source_client = source_client or client
# the cache of existing directories in the destination file share. the cache helps to avoid
# repeatedly create existing directory so as to optimize the performance.
existing_dirs = set([])
if not source_sas and client.account_name != source_client.account_name:
# when file is copied across storage account without sas, generate a short lived
# sas for it
source_sas = create_short_lived_share_sas(source_client.account_name,
source_client.account_key,
source_share)
def action_file_copy(file_info):
dir_name, file_name = file_info
if dryrun:
logger.warning(' - copy file %s', os.path.join(dir_name, file_name))
else:
return _create_file_and_directory_from_file(
client, source_client, destination_share, source_share, source_sas, dir_name,
file_name, destination_dir=destination_path, metadata=metadata,
timeout=timeout, existing_dirs=existing_dirs)
return list(filter_none(action_file_copy(file) for file in
collect_files(source_client, source_share, pattern)))
else:
# won't happen, the validator should ensure either source_container or source_share is set
raise ValueError('Fail to find source. Neither blob container or file share is specified.')
def storage_file_copy_batch(cmd,
client,
source_client,
destination_share=None,
destination_path=None,
source_container=None,
source_share=None,
source_sas=None,
pattern=None,
dryrun=False,
metadata=None,
timeout=None):
"""
Copy a group of files asynchronously
"""
logger = None
if dryrun:
logger = get_logger(__name__)
logger.warning('copy files or blobs to file share')
logger.warning(' account %s', client.account_name)
logger.warning(' share %s', destination_share)
logger.warning(' path %s', destination_path)
logger.warning(' source %s', source_container or source_share)
logger.warning('source type %s',
'blob' if source_container else 'file')
logger.warning(' pattern %s', pattern)
logger.warning(' operations')
if source_container:
# copy blobs to file share
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_blob_service_from_storage_client(
cmd, client)
# the cache of existing directories in the destination file share. the cache helps to avoid
# repeatedly create existing directory so as to optimize the performance.
existing_dirs = set([])
if not source_sas:
source_sas = create_short_lived_container_sas(
cmd, source_client.account_name, source_client.account_key,
source_container)
# pylint: disable=inconsistent-return-statements
def action_blob_copy(blob_name):
if dryrun:
logger.warning(' - copy blob %s', blob_name)
else:
return _create_file_and_directory_from_blob(
client,
source_client,
destination_share,
source_container,
source_sas,
blob_name,
destination_dir=destination_path,
metadata=metadata,
timeout=timeout,
existing_dirs=existing_dirs)
return list(
filter_none(
action_blob_copy(blob) for blob in collect_blobs(
source_client, source_container, pattern)))
if source_share:
# copy files from share to share
# if the source client is None, assume the file share is in the same storage account as
# destination, therefore client is reused.
source_client = source_client or client
# the cache of existing directories in the destination file share. the cache helps to avoid
# repeatedly create existing directory so as to optimize the performance.
existing_dirs = set([])
if not source_sas:
source_sas = create_short_lived_share_sas(
cmd, source_client.account_name, source_client.account_key,
source_share)
# pylint: disable=inconsistent-return-statements
def action_file_copy(file_info):
dir_name, file_name = file_info
if dryrun:
logger.warning(' - copy file %s',
os.path.join(dir_name, file_name))
else:
return _create_file_and_directory_from_file(
client,
source_client,
destination_share,
source_share,
source_sas,
dir_name,
file_name,
destination_dir=destination_path,
metadata=metadata,
timeout=timeout,
existing_dirs=existing_dirs)
return list(
filter_none(
action_file_copy(file) for file in collect_files(
cmd, source_client, source_share, pattern)))
# won't happen, the validator should ensure either source_container or source_share is set
raise ValueError(
'Fail to find source. Neither blob container or file share is specified.'
)
def storage_blob_copy_batch(client,
source_client,
destination_container=None,
source_container=None,
source_share=None,
source_sas=None,
pattern=None,
dryrun=False):
"""Copy a group of blob or files to a blob container."""
logger = None
if dryrun:
logger = get_az_logger(__name__)
logger.warning('copy files or blobs to blob container')
logger.warning(' account %s', client.account_name)
logger.warning(' container %s', destination_container)
logger.warning(' source %s', source_container or source_share)
logger.warning('source type %s',
'blob' if source_container else 'file')
logger.warning(' pattern %s', pattern)
logger.warning(' operations')
if source_container:
# copy blobs for blob container
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_blob_service_from_storage_client(
client)
if not source_sas and client.account_name != source_client.account_name:
# when the blob is copied across storage account without sas, generate a short lived
# sas for it
source_sas = create_short_lived_container_sas(
source_client.account_name, source_client.account_key,
source_container)
def action_blob_copy(blob_name):
if dryrun:
logger.warning(' - copy blob %s', blob_name)
else:
return _copy_blob_to_blob_container(client, source_client,
destination_container,
source_container,
source_sas, blob_name)
return list(
filter_none(
action_blob_copy(blob) for blob in collect_blobs(
source_client, source_container, pattern)))
elif source_share:
# copy blob from file share
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_file_share_from_storage_client(
client)
if not source_sas and client.account_name != source_client.account_name:
# when the file is copied across storage account without sas, generate a short lived sas
source_sas = create_short_lived_share_sas(
source_client.account_name, source_client.account_key,
source_share)
def action_file_copy(file_info):
dir_name, file_name = file_info
if dryrun:
logger.warning(' - copy file %s',
os.path.join(dir_name, file_name))
else:
return _copy_file_to_blob_container(client, source_client,
destination_container,
source_share, source_sas,
dir_name, file_name)
return list(
filter_none(
action_file_copy(file) for file in collect_files(
source_client, source_share, pattern)))
else:
raise ValueError(
'Fail to find source. Neither blob container or file share is specified'
)
def storage_blob_copy_batch(cmd, client, source_client, destination_container=None,
destination_path=None, source_container=None, source_share=None,
source_sas=None, pattern=None, dryrun=False):
"""Copy a group of blob or files to a blob container."""
logger = None
if dryrun:
logger = get_logger(__name__)
logger.warning('copy files or blobs to blob container')
logger.warning(' account %s', client.account_name)
logger.warning(' container %s', destination_container)
logger.warning(' source %s', source_container or source_share)
logger.warning('source type %s', 'blob' if source_container else 'file')
logger.warning(' pattern %s', pattern)
logger.warning(' operations')
if source_container:
# copy blobs for blob container
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_blob_service_from_storage_client(cmd, client)
if not source_sas:
source_sas = create_short_lived_container_sas(cmd, source_client.account_name, source_client.account_key,
source_container)
# pylint: disable=inconsistent-return-statements
def action_blob_copy(blob_name):
if dryrun:
logger.warning(' - copy blob %s', blob_name)
else:
return _copy_blob_to_blob_container(client, source_client, destination_container, destination_path,
source_container, source_sas, blob_name)
return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client,
source_container,
pattern)))
elif source_share:
# copy blob from file share
# if the source client is None, recreate one from the destination client.
source_client = source_client or create_file_share_from_storage_client(cmd, client)
if not source_sas:
source_sas = create_short_lived_share_sas(cmd, source_client.account_name, source_client.account_key,
source_share)
# pylint: disable=inconsistent-return-statements
def action_file_copy(file_info):
dir_name, file_name = file_info
if dryrun:
logger.warning(' - copy file %s', os.path.join(dir_name, file_name))
else:
return _copy_file_to_blob_container(client, source_client, destination_container, destination_path,
source_share, source_sas, dir_name, file_name)
return list(filter_none(action_file_copy(file) for file in collect_files(cmd,
source_client,
source_share,
pattern)))
else:
raise ValueError('Fail to find source. Neither blob container or file share is specified')
