def storage_blob_copy_batch(cmd, client, source_client, container_name=None, destination_path=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False): """Copy a group of blob or files to a blob container.""" if dryrun: logger.warning('copy files or blobs to blob container') logger.warning(' account %s', client.account_name) logger.warning(' container %s', container_name) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') source_sas = source_sas.lstrip('?') if source_sas else source_sas if source_container: # copy blobs for blob container # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client(cmd, client) if not source_sas: source_sas = create_short_lived_container_sas(cmd, source_client.account_name, source_client.account_key, source_container) # pylint: disable=inconsistent-return-statements def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _copy_blob_to_blob_container(client, source_client, container_name, destination_path, source_container, source_sas, blob_name) return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, source_container, pattern))) if source_share: # copy blob from file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_file_share_from_storage_client(cmd, client) if not source_sas: source_sas = create_short_lived_share_sas(cmd, source_client.account_name, source_client.account_key, source_share) # pylint: disable=inconsistent-return-statements def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _copy_file_to_blob_container(client, source_client, container_name, destination_path, source_share, source_sas, dir_name, file_name) return list(filter_none(action_file_copy(file) for file in collect_files(cmd, source_client, source_share, pattern))) raise ValueError('Fail to find source. Neither blob container or file share is specified')
def storage_file_copy_batch(client, source_client, destination_share=None, destination_path=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False, metadata=None, timeout=None): """ Copy a group of files asynchronously """ logger = None if dryrun: logger = get_az_logger(__name__) logger.warning('copy files or blobs to file share') logger.warning(' account %s', client.account_name) logger.warning(' share %s', destination_share) logger.warning(' path %s', destination_path) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') if source_container: # copy blobs to file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client(client) # the cache of existing directories in the destination file share. the cache helps to avoid # repeatedly create existing directory so as to optimize the performance. existing_dirs = set([]) if not source_sas and client.account_name != source_client.account_name: # when blob is copied across storage account without sas, generate a short lived # sas for it source_sas = create_short_lived_container_sas(source_client.account_name, source_client.account_key, source_container) def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _create_file_and_directory_from_blob( client, source_client, destination_share, source_container, source_sas, blob_name, destination_dir=destination_path, metadata=metadata, timeout=timeout, existing_dirs=existing_dirs) return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, source_container, pattern))) elif source_share: # copy files from share to share # if the source client is None, assume the file share is in the same storage account as # destination, therefore client is reused. source_client = source_client or client # the cache of existing directories in the destination file share. the cache helps to avoid # repeatedly create existing directory so as to optimize the performance. existing_dirs = set([]) if not source_sas and client.account_name != source_client.account_name: # when file is copied across storage account without sas, generate a short lived # sas for it source_sas = create_short_lived_share_sas(source_client.account_name, source_client.account_key, source_share) def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _create_file_and_directory_from_file( client, source_client, destination_share, source_share, source_sas, dir_name, file_name, destination_dir=destination_path, metadata=metadata, timeout=timeout, existing_dirs=existing_dirs) return list(filter_none(action_file_copy(file) for file in collect_files(source_client, source_share, pattern))) else: # won't happen, the validator should ensure either source_container or source_share is set raise ValueError('Fail to find source. Neither blob container or file share is specified.')
def storage_file_copy_batch(cmd, client, source_client, destination_share=None, destination_path=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False, metadata=None, timeout=None): """ Copy a group of files asynchronously """ logger = None if dryrun: logger = get_logger(__name__) logger.warning('copy files or blobs to file share') logger.warning(' account %s', client.account_name) logger.warning(' share %s', destination_share) logger.warning(' path %s', destination_path) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') if source_container: # copy blobs to file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client( cmd, client) # the cache of existing directories in the destination file share. the cache helps to avoid # repeatedly create existing directory so as to optimize the performance. existing_dirs = set([]) if not source_sas: source_sas = create_short_lived_container_sas( cmd, source_client.account_name, source_client.account_key, source_container) # pylint: disable=inconsistent-return-statements def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _create_file_and_directory_from_blob( client, source_client, destination_share, source_container, source_sas, blob_name, destination_dir=destination_path, metadata=metadata, timeout=timeout, existing_dirs=existing_dirs) return list( filter_none( action_blob_copy(blob) for blob in collect_blobs( source_client, source_container, pattern))) if source_share: # copy files from share to share # if the source client is None, assume the file share is in the same storage account as # destination, therefore client is reused. source_client = source_client or client # the cache of existing directories in the destination file share. the cache helps to avoid # repeatedly create existing directory so as to optimize the performance. existing_dirs = set([]) if not source_sas: source_sas = create_short_lived_share_sas( cmd, source_client.account_name, source_client.account_key, source_share) # pylint: disable=inconsistent-return-statements def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _create_file_and_directory_from_file( client, source_client, destination_share, source_share, source_sas, dir_name, file_name, destination_dir=destination_path, metadata=metadata, timeout=timeout, existing_dirs=existing_dirs) return list( filter_none( action_file_copy(file) for file in collect_files( cmd, source_client, source_share, pattern))) # won't happen, the validator should ensure either source_container or source_share is set raise ValueError( 'Fail to find source. Neither blob container or file share is specified.' )
def storage_blob_copy_batch(client, source_client, destination_container=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False): """Copy a group of blob or files to a blob container.""" logger = None if dryrun: logger = get_az_logger(__name__) logger.warning('copy files or blobs to blob container') logger.warning(' account %s', client.account_name) logger.warning(' container %s', destination_container) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') if source_container: # copy blobs for blob container # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client( client) if not source_sas and client.account_name != source_client.account_name: # when the blob is copied across storage account without sas, generate a short lived # sas for it source_sas = create_short_lived_container_sas( source_client.account_name, source_client.account_key, source_container) def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _copy_blob_to_blob_container(client, source_client, destination_container, source_container, source_sas, blob_name) return list( filter_none( action_blob_copy(blob) for blob in collect_blobs( source_client, source_container, pattern))) elif source_share: # copy blob from file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_file_share_from_storage_client( client) if not source_sas and client.account_name != source_client.account_name: # when the file is copied across storage account without sas, generate a short lived sas source_sas = create_short_lived_share_sas( source_client.account_name, source_client.account_key, source_share) def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _copy_file_to_blob_container(client, source_client, destination_container, source_share, source_sas, dir_name, file_name) return list( filter_none( action_file_copy(file) for file in collect_files( source_client, source_share, pattern))) else: raise ValueError( 'Fail to find source. Neither blob container or file share is specified' )
def storage_blob_copy_batch(cmd, client, source_client, destination_container=None, destination_path=None, source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False): """Copy a group of blob or files to a blob container.""" logger = None if dryrun: logger = get_logger(__name__) logger.warning('copy files or blobs to blob container') logger.warning(' account %s', client.account_name) logger.warning(' container %s', destination_container) logger.warning(' source %s', source_container or source_share) logger.warning('source type %s', 'blob' if source_container else 'file') logger.warning(' pattern %s', pattern) logger.warning(' operations') if source_container: # copy blobs for blob container # if the source client is None, recreate one from the destination client. source_client = source_client or create_blob_service_from_storage_client(cmd, client) if not source_sas: source_sas = create_short_lived_container_sas(cmd, source_client.account_name, source_client.account_key, source_container) # pylint: disable=inconsistent-return-statements def action_blob_copy(blob_name): if dryrun: logger.warning(' - copy blob %s', blob_name) else: return _copy_blob_to_blob_container(client, source_client, destination_container, destination_path, source_container, source_sas, blob_name) return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, source_container, pattern))) elif source_share: # copy blob from file share # if the source client is None, recreate one from the destination client. source_client = source_client or create_file_share_from_storage_client(cmd, client) if not source_sas: source_sas = create_short_lived_share_sas(cmd, source_client.account_name, source_client.account_key, source_share) # pylint: disable=inconsistent-return-statements def action_file_copy(file_info): dir_name, file_name = file_info if dryrun: logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) else: return _copy_file_to_blob_container(client, source_client, destination_container, destination_path, source_share, source_sas, dir_name, file_name) return list(filter_none(action_file_copy(file) for file in collect_files(cmd, source_client, source_share, pattern))) else: raise ValueError('Fail to find source. Neither blob container or file share is specified')