def storage_blob_upload_batch(cmd, client, source, destination, pattern=None, # pylint: disable=too-many-locals
source_files=None, destination_path=None,
destination_container_name=None, blob_type=None,
content_settings=None, metadata=None, validate_content=False,
maxsize_condition=None, max_connections=2, lease_id=None, progress_callback=None,
if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None, dryrun=False):
def _create_return_result(blob_name, blob_content_settings, upload_result=None):
blob_name = normalize_blob_file_path(destination_path, blob_name)
return {
'Blob': client.make_blob_url(destination_container_name, blob_name),
'Type': blob_content_settings.content_type,
'Last Modified': upload_result.last_modified if upload_result else None,
'eTag': upload_result.etag if upload_result else None}
logger = get_logger(__name__)
t_content_settings = cmd.get_models('blob.models#ContentSettings')
results = []
if dryrun:
logger.info('upload action: from %s to %s', source, destination)
logger.info(' pattern %s', pattern)
logger.info(' container %s', destination_container_name)
logger.info(' type %s', blob_type)
logger.info(' total %d', len(source_files))
results = []
for src, dst in source_files or []:
results.append(_create_return_result(dst, guess_content_type(src, content_settings, t_content_settings)))
else:
@check_precondition_success
def _upload_blob(*args, **kwargs):
return upload_blob(*args, **kwargs)
for src, dst in source_files or []:
logger.warning('uploading %s', src)
guessed_content_settings = guess_content_type(src, content_settings, t_content_settings)
include, result = _upload_blob(cmd, client, destination_container_name,
normalize_blob_file_path(destination_path, dst), src,
blob_type=blob_type, content_settings=guessed_content_settings,
metadata=metadata, validate_content=validate_content,
maxsize_condition=maxsize_condition, max_connections=max_connections,
lease_id=lease_id, progress_callback=progress_callback,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since, if_match=if_match,
if_none_match=if_none_match, timeout=timeout)
if include:
results.append(_create_return_result(dst, guessed_content_settings, result))
num_failures = len(source_files) - len(results)
if num_failures:
logger.warning('%s of %s files not uploaded due to "Failed Precondition"', num_failures, len(source_files))
return results
def _upload_action(src, dst):
dst = normalize_blob_file_path(destination_path, dst)
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {
'share_name':
destination,
'directory_name':
dir_name,
'file_name':
file_name,
'local_file_path':
src,
'progress_callback':
progress_callback,
'content_settings':
guess_content_type(src, content_settings, settings_class),
'metadata':
metadata,
'max_connections':
max_connections
}
if cmd.supported_api_version(min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
def validator(cmd, namespace):
t_blob_content_settings = cmd.get_models(
'_models#ContentSettings', resource_type=CUSTOM_DATA_STORAGE_BLOB)
# must run certain validators first for an update
if update:
validate_client_parameters(cmd, namespace)
ns = vars(namespace)
clear_content_settings = ns.pop('clear_content_settings', False)
# retrieve the existing object properties for an update
if update and not clear_content_settings:
account = ns.get('account_name')
key = ns.get('account_key')
cs = ns.get('connection_string')
sas = ns.get('sas_token')
token_credential = ns.get('token_credential')
if _class_name(settings_class) == _class_name(
t_blob_content_settings):
container = ns.get('container_name')
blob = ns.get('blob_name')
lease_id = ns.get('lease_id')
client = cf_blob_client(cmd.cli_ctx,
connection_string=cs,
account_name=account,
account_key=key,
token_credential=token_credential,
sas_token=sas,
container=container,
blob=blob)
props = client.get_blob_properties(
lease=lease_id).content_settings
# create new properties
new_props = settings_class(
content_type=ns.pop('content_type', None),
content_disposition=ns.pop('content_disposition', None),
content_encoding=ns.pop('content_encoding', None),
content_language=ns.pop('content_language', None),
content_md5=ns.pop('content_md5', None),
cache_control=ns.pop('content_cache_control', None))
# if update, fill in any None values with existing
if update:
if not clear_content_settings:
for attr in [
'content_type', 'content_disposition',
'content_encoding', 'content_language', 'content_md5',
'cache_control'
]:
if getattr(new_props, attr) is None:
setattr(new_props, attr, getattr(props, attr))
else:
if guess_from_file:
new_props = guess_content_type(ns[guess_from_file], new_props,
settings_class)
ns['content_settings'] = new_props
def _upload_action(src, dst):
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {
'share_name':
destination,
'directory_name':
dir_name,
'file_name':
file_name,
'local_file_path':
src,
'content_settings':
guess_content_type(src, content_settings, settings_class),
'metadata':
metadata,
'max_connections':
max_connections,
}
if supported_api_version(ResourceType.DATA_STORAGE,
min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
def validator(cmd, namespace):
t_base_blob_service, t_file_service, t_blob_content_settings, t_file_content_settings = cmd.get_models(
'blob.baseblobservice#BaseBlobService',
'file#FileService',
'blob.models#ContentSettings',
'file.models#ContentSettings')
# must run certain validators first for an update
if update:
validate_client_parameters(cmd, namespace)
if update and _class_name(settings_class) == _class_name(t_file_content_settings):
get_file_path_validator()(namespace)
ns = vars(namespace)
# retrieve the existing object properties for an update
if update:
account = ns.get('account_name')
key = ns.get('account_key')
cs = ns.get('connection_string')
sas = ns.get('sas_token')
if _class_name(settings_class) == _class_name(t_blob_content_settings):
client = get_storage_data_service_client(cmd.cli_ctx,
t_base_blob_service,
account,
key,
cs,
sas)
container = ns.get('container_name')
blob = ns.get('blob_name')
lease_id = ns.get('lease_id')
props = client.get_blob_properties(container, blob, lease_id=lease_id).properties.content_settings
elif _class_name(settings_class) == _class_name(t_file_content_settings):
client = get_storage_data_service_client(cmd.cli_ctx, t_file_service, account, key, cs, sas)
share = ns.get('share_name')
directory = ns.get('directory_name')
filename = ns.get('file_name')
props = client.get_file_properties(share, directory, filename).properties.content_settings
# create new properties
new_props = settings_class(
content_type=ns.pop('content_type', None),
content_disposition=ns.pop('content_disposition', None),
content_encoding=ns.pop('content_encoding', None),
content_language=ns.pop('content_language', None),
content_md5=ns.pop('content_md5', None),
cache_control=ns.pop('content_cache_control', None)
)
# if update, fill in any None values with existing
if update:
for attr in ['content_type', 'content_disposition', 'content_encoding', 'content_language', 'content_md5',
'cache_control']:
if getattr(new_props, attr) is None:
setattr(new_props, attr, getattr(props, attr))
else:
if guess_from_file:
new_props = guess_content_type(ns[guess_from_file], new_props, settings_class)
ns['content_settings'] = new_props
def storage_blob_upload_batch(cmd, client, source, destination, pattern=None, # pylint: disable=too-many-locals
source_files=None,
destination_container_name=None, blob_type=None,
content_settings=None, metadata=None, validate_content=False,
maxsize_condition=None, max_connections=2, lease_id=None,
if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None, dryrun=False):
def _create_return_result(blob_name, blob_content_settings, upload_result=None):
return {
'Blob': client.make_blob_url(destination_container_name, blob_name),
'Type': blob_content_settings.content_type,
'Last Modified': upload_result.last_modified if upload_result else None,
'eTag': upload_result.etag if upload_result else None}
logger = get_logger(__name__)
t_content_settings = cmd.get_models('blob.models#ContentSettings')
results = []
if dryrun:
logger.info('upload action: from %s to %s', source, destination)
logger.info(' pattern %s', pattern)
logger.info(' container %s', destination_container_name)
logger.info(' type %s', blob_type)
logger.info(' total %d', len(source_files))
results = []
for src, dst in source_files or []:
results.append(_create_return_result(dst, guess_content_type(src, content_settings, t_content_settings)))
else:
for src, dst in source_files or []:
logger.warning('uploading %s', src)
guessed_content_settings = guess_content_type(src, content_settings, t_content_settings)
result = upload_blob(cmd, client, destination_container_name, dst, src,
blob_type=blob_type, content_settings=guessed_content_settings, metadata=metadata,
validate_content=validate_content, maxsize_condition=maxsize_condition,
max_connections=max_connections, lease_id=lease_id,
if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since,
if_match=if_match, if_none_match=if_none_match, timeout=timeout)
results.append(_create_return_result(dst, guessed_content_settings, result))
return results
def _upload_action(src, dst):
dst = normalize_blob_file_path(destination_path, dst)
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {'share_name': destination, 'directory_name': dir_name, 'file_name': file_name,
'local_file_path': src, 'progress_callback': progress_callback,
'content_settings': guess_content_type(src, content_settings, settings_class),
'metadata': metadata, 'max_connections': max_connections}
if cmd.supported_api_version(min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
def storage_file_upload_batch(client, destination, source, pattern=None, dryrun=False, validate_content=False,
content_settings=None, max_connections=1, metadata=None):
""" Upload local files to Azure Storage File Share in batch """
from .util import glob_files_locally
source_files = [c for c in glob_files_locally(source, pattern)]
logger = get_az_logger(__name__)
settings_class = get_sdk(ResourceType.DATA_STORAGE, 'file.models#ContentSettings')
if dryrun:
logger.info('upload files to file share')
logger.info(' account %s', client.account_name)
logger.info(' share %s', destination)
logger.info(' total %d', len(source_files or []))
return [{'File': client.make_file_url(destination, os.path.dirname(src), os.path.basename(dst)),
'Type': guess_content_type(src, content_settings, settings_class).content_type}
for src, dst in source_files]
# TODO: Performance improvement
# 1. Upload files in parallel
def _upload_action(src, dst):
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {
'share_name': destination,
'directory_name': dir_name,
'file_name': file_name,
'local_file_path': src,
'content_settings': guess_content_type(src, content_settings, settings_class),
'metadata': metadata,
'max_connections': max_connections,
}
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
return list(_upload_action(src, dst) for src, dst in source_files)
def storage_file_upload_batch(client, destination, source, pattern=None, dryrun=False, validate_content=False,
content_settings=None, max_connections=1, metadata=None):
""" Upload local files to Azure Storage File Share in batch """
from .util import glob_files_locally
source_files = [c for c in glob_files_locally(source, pattern)]
logger = get_az_logger(__name__)
settings_class = get_sdk(ResourceType.DATA_STORAGE, 'file.models#ContentSettings')
if dryrun:
logger.info('upload files to file share')
logger.info(' account %s', client.account_name)
logger.info(' share %s', destination)
logger.info(' total %d', len(source_files))
return [{'File': client.make_file_url(destination, os.path.dirname(src), os.path.basename(dst)),
'Type': guess_content_type(src, content_settings, settings_class).content_type}
for src, dst in source_files]
# TODO: Performance improvement
# 1. Upload files in parallel
def _upload_action(src, dst):
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {
'share_name': destination,
'directory_name': dir_name,
'file_name': file_name,
'local_file_path': src,
'content_settings': guess_content_type(src, content_settings, settings_class),
'metadata': metadata,
'max_connections': max_connections,
}
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
return list(_upload_action(src, dst) for src, dst in source_files)
def storage_file_upload_batch(cmd, client, destination, source, destination_path=None, pattern=None, dryrun=False,
validate_content=False, content_settings=None, max_connections=1, metadata=None,
progress_callback=None):
""" Upload local files to Azure Storage File Share in batch """
from azure.cli.command_modules.storage.util import glob_files_locally, normalize_blob_file_path
source_files = list(glob_files_locally(source, pattern))
logger = get_logger(__name__)
settings_class = cmd.get_models('file.models#ContentSettings')
if dryrun:
logger.info('upload files to file share')
logger.info(' account %s', client.account_name)
logger.info(' share %s', destination)
logger.info(' total %d', len(source_files))
return [{'File': client.make_file_url(destination, os.path.dirname(dst) or None, os.path.basename(dst)),
'Type': guess_content_type(src, content_settings, settings_class).content_type} for src, dst in
source_files]
# TODO: Performance improvement
# 1. Upload files in parallel
def _upload_action(src, dst):
dst = normalize_blob_file_path(destination_path, dst)
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {'share_name': destination, 'directory_name': dir_name, 'file_name': file_name,
'local_file_path': src, 'progress_callback': progress_callback,
'content_settings': guess_content_type(src, content_settings, settings_class),
'metadata': metadata, 'max_connections': max_connections}
if cmd.supported_api_version(min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
return list(_upload_action(src, dst) for src, dst in source_files)
def _upload_action(src, dst):
dir_name = os.path.dirname(dst)
file_name = os.path.basename(dst)
_make_directory_in_files_share(client, destination, dir_name)
create_file_args = {
'share_name': destination,
'directory_name': dir_name,
'file_name': file_name,
'local_file_path': src,
'content_settings': guess_content_type(src, content_settings, settings_class),
'metadata': metadata,
'max_connections': max_connections,
}
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'):
create_file_args['validate_content'] = validate_content
logger.warning('uploading %s', src)
client.create_file_from_path(**create_file_args)
return client.make_file_url(destination, dir_name, file_name)
def validator(cmd, namespace):
t_blob_content_settings = cmd.get_models(
'_models#ContentSettings', resource_type=CUSTOM_DATA_STORAGE_BLOB)
# must run certain validators first for an update
if update:
validate_client_parameters(cmd, namespace)
ns = vars(namespace)
clear_content_settings = ns.pop('clear_content_settings', False)
# retrieve the existing object properties for an update
if update and not clear_content_settings:
account = ns.get('account_name')
key = ns.get('account_key')
cs = ns.get('connection_string')
sas = ns.get('sas_token')
token_credential = ns.get('token_credential')
if _class_name(settings_class) == _class_name(
t_blob_content_settings):
container = ns.get('container_name')
blob = ns.get('blob_name')
lease_id = ns.get('lease_id')
client_kwargs = {
'connection_string': cs,
'account_name': account,
'account_key': key,
'token_credential': token_credential,
'sas_token': sas,
'container_name': container,
'blob_name': blob
}
client = cf_blob_client(cmd.cli_ctx, client_kwargs)
props = client.get_blob_properties(
lease=lease_id).content_settings
# create new properties
new_props = settings_class(
content_type=ns.pop('content_type', None),
content_disposition=ns.pop('content_disposition', None),
content_encoding=ns.pop('content_encoding', None),
content_language=ns.pop('content_language', None),
content_md5=ns.pop('content_md5', None),
cache_control=ns.pop('content_cache_control', None))
# if update, fill in any None values with existing
if update:
if not clear_content_settings:
for attr in [
'content_type', 'content_disposition',
'content_encoding', 'content_language', 'content_md5',
'cache_control'
]:
if getattr(new_props, attr) is None:
setattr(new_props, attr, getattr(props, attr))
else:
if guess_from_file:
new_props = guess_content_type(ns[guess_from_file], new_props,
settings_class)
# In track2 SDK, the content_md5 type should be bytearray. And then it will serialize to a string for request.
# To keep consistent with track1 input and CLI will treat all parameter values as string. Here is to transform
# content_md5 value to bytearray. And track2 SDK will serialize it into the right value with str type in header.
if process_md5 and new_props.content_md5:
from .track2_util import _str_to_bytearray
new_props.content_md5 = _str_to_bytearray(new_props.content_md5)
ns['content_settings'] = new_props
def upload_blob(cmd, client, file_path, container_name=None, blob_name=None, blob_type=None, content_settings=None,
metadata=None, validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None,
tier=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None,
timeout=None, progress_callback=None, encryption_scope=None):
"""Upload a blob to a container."""
if encryption_scope:
count = os.path.getsize(file_path)
with open(file_path, 'rb') as stream:
data = stream.read(count)
from azure.core import MatchConditions
upload_args = {
'content_settings': content_settings,
'metadata': metadata,
'timeout': timeout,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'blob_type': transform_blob_type(cmd, blob_type),
'validate_content': validate_content,
'lease': lease_id,
'max_concurrency': max_connections,
}
if cmd.supported_api_version(min_api='2017-04-17') and tier:
upload_args['premium_page_blob_tier'] = tier
if maxsize_condition:
upload_args['maxsize_condition'] = maxsize_condition
if cmd.supported_api_version(min_api='2016-05-31'):
upload_args['validate_content'] = validate_content
# Precondition Check
if if_match:
if if_match == '*':
upload_args['match_condition'] = MatchConditions.IfPresent
else:
upload_args['etag'] = if_match
upload_args['match_condition'] = MatchConditions.IfNotModified
if if_none_match:
upload_args['etag'] = if_none_match
upload_args['match_condition'] = MatchConditions.IfModified
response = client.upload_blob(data=data, length=count, encryption_scope=encryption_scope, **upload_args)
if response['content_md5'] is not None:
from msrest import Serializer
response['content_md5'] = Serializer.serialize_bytearray(response['content_md5'])
return response
t_content_settings = cmd.get_models('blob.models#ContentSettings')
content_settings = guess_content_type(file_path, content_settings, t_content_settings)
def upload_append_blob():
check_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if client.exists(container_name, blob_name):
# used to check for the preconditions as append_blob_from_path() cannot
client.get_blob_properties(**check_blob_args)
else:
client.create_blob(content_settings=content_settings, metadata=metadata, **check_blob_args)
append_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': progress_callback,
'maxsize_condition': maxsize_condition,
'lease_id': lease_id,
'timeout': timeout
}
if cmd.supported_api_version(min_api='2016-05-31'):
append_blob_args['validate_content'] = validate_content
return client.append_blob_from_path(**append_blob_args)
def upload_block_blob():
# increase the block size to 100MB when the block list will contain more than 50,000 blocks
if os.path.isfile(file_path) and os.stat(file_path).st_size > 50000 * 4 * 1024 * 1024:
client.MAX_BLOCK_SIZE = 100 * 1024 * 1024
client.MAX_SINGLE_PUT_SIZE = 256 * 1024 * 1024
create_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': progress_callback,
'content_settings': content_settings,
'metadata': metadata,
'max_connections': max_connections,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if cmd.supported_api_version(min_api='2017-04-17') and tier:
create_blob_args['premium_page_blob_tier'] = tier
if cmd.supported_api_version(min_api='2016-05-31'):
create_blob_args['validate_content'] = validate_content
return client.create_blob_from_path(**create_blob_args)
type_func = {
'append': upload_append_blob,
'block': upload_block_blob,
'page': upload_block_blob # same implementation
}
return type_func[blob_type]()
def upload_blob(client, container_name, blob_name, file_path, blob_type=None, content_settings=None, metadata=None,
validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, tier=None,
if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None):
"""Upload a blob to a container."""
settings_class = get_sdk(ResourceType.DATA_STORAGE, 'blob.models#ContentSettings')
content_settings = guess_content_type(file_path, content_settings, settings_class)
def upload_append_blob():
if not client.exists(container_name, blob_name):
client.create_blob(
container_name=container_name,
blob_name=blob_name,
content_settings=content_settings,
metadata=metadata,
lease_id=lease_id,
if_modified_since=if_modified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout)
append_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': _update_progress,
'maxsize_condition': maxsize_condition,
'lease_id': lease_id,
'timeout': timeout
}
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'):
append_blob_args['validate_content'] = validate_content
return client.append_blob_from_path(**append_blob_args)
def upload_block_blob():
import os
# increase the block size to 100MB when the file is larger than 200GB
if os.path.isfile(file_path) and os.stat(file_path).st_size > 200 * 1024 * 1024 * 1024:
client.MAX_BLOCK_SIZE = 100 * 1024 * 1024
client.MAX_SINGLE_PUT_SIZE = 256 * 1024 * 1024
create_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': _update_progress,
'content_settings': content_settings,
'metadata': metadata,
'max_connections': max_connections,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2017-04-17') and tier:
create_blob_args['premium_page_blob_tier'] = tier
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'):
create_blob_args['validate_content'] = validate_content
return client.create_blob_from_path(**create_blob_args)
type_func = {
'append': upload_append_blob,
'block': upload_block_blob,
'page': upload_block_blob # same implementation
}
return type_func[blob_type]()
def upload_blob(cmd, client, container_name, blob_name, file_path, blob_type=None, content_settings=None, metadata=None,
validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, tier=None,
if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None,
progress_callback=None):
"""Upload a blob to a container."""
t_content_settings = cmd.get_models('blob.models#ContentSettings')
content_settings = guess_content_type(file_path, content_settings, t_content_settings)
def upload_append_blob():
check_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if client.exists(container_name, blob_name):
# used to check for the preconditions as append_blob_from_path() cannot
client.get_blob_properties(**check_blob_args)
else:
client.create_blob(content_settings=content_settings, metadata=metadata, **check_blob_args)
append_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': progress_callback,
'maxsize_condition': maxsize_condition,
'lease_id': lease_id,
'timeout': timeout
}
if cmd.supported_api_version(min_api='2016-05-31'):
append_blob_args['validate_content'] = validate_content
return client.append_blob_from_path(**append_blob_args)
def upload_block_blob():
# increase the block size to 100MB when the block list will contain more than 50,000 blocks
if os.path.isfile(file_path) and os.stat(file_path).st_size > 50000 * 4 * 1024 * 1024:
client.MAX_BLOCK_SIZE = 100 * 1024 * 1024
client.MAX_SINGLE_PUT_SIZE = 256 * 1024 * 1024
create_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': progress_callback,
'content_settings': content_settings,
'metadata': metadata,
'max_connections': max_connections,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if cmd.supported_api_version(min_api='2017-04-17') and tier:
create_blob_args['premium_page_blob_tier'] = tier
if cmd.supported_api_version(min_api='2016-05-31'):
create_blob_args['validate_content'] = validate_content
return client.create_blob_from_path(**create_blob_args)
type_func = {
'append': upload_append_blob,
'block': upload_block_blob,
'page': upload_block_blob # same implementation
}
return type_func[blob_type]()
def storage_blob_upload_batch(
client,
source,
destination,
pattern=None,
source_files=None, # pylint: disable=too-many-locals
destination_container_name=None,
blob_type=None,
content_settings=None,
metadata=None,
validate_content=False,
maxsize_condition=None,
max_connections=2,
lease_id=None,
if_modified_since=None,
if_unmodified_since=None,
if_match=None,
if_none_match=None,
timeout=None,
dryrun=False):
"""
Upload files to storage container as blobs
:param str source:
The directory where the files to be uploaded.
:param str destination:
The string represents the destination of this upload operation. The source can be the
container URL or the container name. When the source is the container URL, the storage
account name will parsed from the URL.
:param str pattern:
The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]',
and '[!seq]'.
:param bool dryrun:
Show the summary of the operations to be taken instead of actually upload the file(s)
:param string if_match:
An ETag value, or the wildcard character (*). Specify this header to perform the operation
only if the resource's ETag matches the value specified.
:param string if_none_match:
An ETag value, or the wildcard character (*). Specify this header to perform the
operation only if the resource's ETag does not match the value specified. Specify the
wildcard character (*) to perform the operation only if the resource does not exist,
and fail the operation if it does exist.
"""
def _append_blob(file_path, blob_name, blob_content_settings):
if not client.exists(destination_container_name, blob_name):
client.create_blob(container_name=destination_container_name,
blob_name=blob_name,
content_settings=blob_content_settings,
metadata=metadata,
lease_id=lease_id,
if_modified_since=if_modified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout)
append_blob_args = {
'container_name': destination_container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': lambda c, t: None,
'maxsize_condition': maxsize_condition,
'lease_id': lease_id,
'timeout': timeout
}
if supported_api_version(ResourceType.DATA_STORAGE,
min_api='2016-05-31'):
append_blob_args['validate_content'] = validate_content
return client.append_blob_from_path(**append_blob_args)
def _upload_blob(file_path, blob_name, blob_content_settings):
create_blob_args = {
'container_name': destination_container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': lambda c, t: None,
'content_settings': blob_content_settings,
'metadata': metadata,
'max_connections': max_connections,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if supported_api_version(ResourceType.DATA_STORAGE,
min_api='2016-05-31'):
create_blob_args['validate_content'] = validate_content
return client.create_blob_from_path(**create_blob_args)
def _create_return_result(blob_name,
blob_content_settings,
upload_result=None):
return {
'Blob': client.make_blob_url(destination_container_name,
blob_name),
'Type': blob_content_settings.content_type,
'Last Modified':
upload_result.last_modified if upload_result else None,
'eTag': upload_result.etag if upload_result else None
}
upload_action = _upload_blob if blob_type == 'block' or blob_type == 'page' else _append_blob
logger = get_az_logger(__name__)
settings_class = get_sdk(ResourceType.DATA_STORAGE,
'blob.models#ContentSettings')
results = []
if dryrun:
logger.info('upload action: from %s to %s', source, destination)
logger.info(' pattern %s', pattern)
logger.info(' container %s', destination_container_name)
logger.info(' type %s', blob_type)
logger.info(' total %d', len(source_files))
results = []
for src, dst in source_files or []:
results.append(
_create_return_result(
dst,
guess_content_type(src, content_settings, settings_class)))
else:
for src, dst in source_files or []:
logger.warning('uploading {}'.format(src))
guessed_content_settings = guess_content_type(
src, content_settings, settings_class)
results.append(
_create_return_result(
dst, guessed_content_settings,
upload_action(src, dst, guessed_content_settings)))
return results
def storage_blob_upload_batch(
cmd,
client,
source,
destination,
pattern=None, # pylint: disable=too-many-locals
source_files=None,
destination_path=None,
destination_container_name=None,
blob_type=None,
content_settings=None,
metadata=None,
validate_content=False,
maxsize_condition=None,
max_connections=2,
lease_id=None,
progress_callback=None,
if_modified_since=None,
if_unmodified_since=None,
if_match=None,
if_none_match=None,
timeout=None,
dryrun=False,
socket_timeout=None,
**kwargs):
def _create_return_result(blob_content_settings, upload_result=None):
return {
'Blob': client.url,
'Type': blob_content_settings.content_type,
'Last Modified':
upload_result['last_modified'] if upload_result else None,
'eTag': upload_result['etag'] if upload_result else None
}
source_files = source_files or []
t_content_settings = cmd.get_models(
'_models#ContentSettings',
resource_type=cmd.command_kwargs['resource_type'])
results = []
if dryrun:
logger.info('upload action: from %s to %s', source, destination)
logger.info(' pattern %s', pattern)
logger.info(' container %s', destination_container_name)
logger.info(' type %s', blob_type)
logger.info(' total %d', len(source_files))
results = []
for src, dst in source_files:
results.append(
_create_return_result(blob_content_settings=guess_content_type(
src, content_settings, t_content_settings)))
else:
@check_precondition_success
def _upload_blob(*args, **kwargs):
return upload_blob(*args, **kwargs)
# Tell progress reporter to reuse the same hook
if progress_callback:
progress_callback.reuse = True
for index, source_file in enumerate(source_files):
src, dst = source_file
# logger.warning('uploading %s', src)
guessed_content_settings = guess_content_type(
src, content_settings, t_content_settings)
# add blob name and number to progress message
if progress_callback:
progress_callback.message = '{}/{}: "{}"'.format(
index + 1, len(source_files),
normalize_blob_file_path(destination_path, dst))
blob_client = client.get_blob_client(
container=destination_container_name,
blob=normalize_blob_file_path(destination_path, dst))
include, result = _upload_blob(
cmd,
blob_client,
file_path=src,
blob_type=blob_type,
content_settings=guessed_content_settings,
metadata=metadata,
validate_content=validate_content,
maxsize_condition=maxsize_condition,
max_connections=max_connections,
lease_id=lease_id,
progress_callback=progress_callback,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout,
**kwargs)
if include:
results.append(
_create_return_result(
blob_content_settings=guessed_content_settings,
upload_result=result))
# end progress hook
if progress_callback:
progress_callback.hook.end()
num_failures = len(source_files) - len(results)
if num_failures:
logger.warning(
'%s of %s files not uploaded due to "Failed Precondition"',
num_failures, len(source_files))
return results
def upload_blob(cmd,
client,
container_name,
blob_name,
file_path,
blob_type=None,
content_settings=None,
metadata=None,
validate_content=False,
maxsize_condition=None,
max_connections=2,
lease_id=None,
tier=None,
if_modified_since=None,
if_unmodified_since=None,
if_match=None,
if_none_match=None,
timeout=None,
progress_callback=None):
"""Upload a blob to a container."""
t_content_settings = cmd.get_models('blob.models#ContentSettings')
content_settings = guess_content_type(file_path, content_settings,
t_content_settings)
def upload_append_blob():
if not client.exists(container_name, blob_name):
client.create_blob(container_name=container_name,
blob_name=blob_name,
content_settings=content_settings,
metadata=metadata,
lease_id=lease_id,
if_modified_since=if_modified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout)
append_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': progress_callback,
'maxsize_condition': maxsize_condition,
'lease_id': lease_id,
'timeout': timeout
}
if cmd.supported_api_version(min_api='2016-05-31'):
append_blob_args['validate_content'] = validate_content
return client.append_blob_from_path(**append_blob_args)
def upload_block_blob():
# increase the block size to 100MB when the file is larger than 200GB
if os.path.isfile(file_path) and os.stat(
file_path).st_size > 200 * 1024 * 1024 * 1024:
client.MAX_BLOCK_SIZE = 100 * 1024 * 1024
client.MAX_SINGLE_PUT_SIZE = 256 * 1024 * 1024
create_blob_args = {
'container_name': container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': progress_callback,
'content_settings': content_settings,
'metadata': metadata,
'max_connections': max_connections,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if cmd.supported_api_version(min_api='2017-04-17') and tier:
create_blob_args['premium_page_blob_tier'] = tier
if cmd.supported_api_version(min_api='2016-05-31'):
create_blob_args['validate_content'] = validate_content
return client.create_blob_from_path(**create_blob_args)
type_func = {
'append': upload_append_blob,
'block': upload_block_blob,
'page': upload_block_blob # same implementation
}
return type_func[blob_type]()
def storage_blob_upload_batch(client, source, destination, pattern=None, source_files=None, # pylint: disable=too-many-locals
destination_container_name=None, blob_type=None,
content_settings=None, metadata=None, validate_content=False,
maxsize_condition=None, max_connections=2, lease_id=None,
if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None, dryrun=False):
"""
Upload files to storage container as blobs
:param str source:
The directory where the files to be uploaded.
:param str destination:
The string represents the destination of this upload operation. The source can be the
container URL or the container name. When the source is the container URL, the storage
account name will parsed from the URL.
:param str pattern:
The pattern is used for files globbing. The supported patterns are '*', '?', '[seq]',
and '[!seq]'.
:param bool dryrun:
Show the summary of the operations to be taken instead of actually upload the file(s)
:param string if_match:
An ETag value, or the wildcard character (*). Specify this header to perform the operation
only if the resource's ETag matches the value specified.
:param string if_none_match:
An ETag value, or the wildcard character (*). Specify this header to perform the
operation only if the resource's ETag does not match the value specified. Specify the
wildcard character (*) to perform the operation only if the resource does not exist,
and fail the operation if it does exist.
"""
def _append_blob(file_path, blob_name, blob_content_settings):
if not client.exists(destination_container_name, blob_name):
client.create_blob(
container_name=destination_container_name,
blob_name=blob_name,
content_settings=blob_content_settings,
metadata=metadata,
lease_id=lease_id,
if_modified_since=if_modified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout)
append_blob_args = {
'container_name': destination_container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': lambda c, t: None,
'maxsize_condition': maxsize_condition,
'lease_id': lease_id,
'timeout': timeout
}
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'):
append_blob_args['validate_content'] = validate_content
return client.append_blob_from_path(**append_blob_args)
def _upload_blob(file_path, blob_name, blob_content_settings):
create_blob_args = {
'container_name': destination_container_name,
'blob_name': blob_name,
'file_path': file_path,
'progress_callback': lambda c, t: None,
'content_settings': blob_content_settings,
'metadata': metadata,
'max_connections': max_connections,
'lease_id': lease_id,
'if_modified_since': if_modified_since,
'if_unmodified_since': if_unmodified_since,
'if_match': if_match,
'if_none_match': if_none_match,
'timeout': timeout
}
if supported_api_version(ResourceType.DATA_STORAGE, min_api='2016-05-31'):
create_blob_args['validate_content'] = validate_content
return client.create_blob_from_path(**create_blob_args)
def _create_return_result(blob_name, blob_content_settings, upload_result=None):
return {
'Blob': client.make_blob_url(destination_container_name, blob_name),
'Type': blob_content_settings.content_type,
'Last Modified': upload_result.last_modified if upload_result else None,
'eTag': upload_result.etag if upload_result else None}
upload_action = _upload_blob if blob_type == 'block' or blob_type == 'page' else _append_blob
logger = get_az_logger(__name__)
settings_class = get_sdk(ResourceType.DATA_STORAGE, 'blob.models#ContentSettings')
results = []
if dryrun:
logger.info('upload action: from %s to %s', source, destination)
logger.info(' pattern %s', pattern)
logger.info(' container %s', destination_container_name)
logger.info(' type %s', blob_type)
logger.info(' total %d', len(source_files))
results = []
for src, dst in source_files or []:
results.append(_create_return_result(dst, guess_content_type(src, content_settings, settings_class)))
else:
for src, dst in source_files or []:
logger.warning('uploading {}'.format(src))
guessed_content_settings = guess_content_type(src, content_settings, settings_class)
results.append(
_create_return_result(dst, guessed_content_settings, upload_action(src, dst, guessed_content_settings)))
return results
