def _validate_and_format_range_headers(request,
start_range,
end_range,
start_range_required=True,
end_range_required=True,
check_content_md5=False,
is_source=False):
# If end range is provided, start range must be provided
if start_range_required or end_range is not None:
_validate_not_none('start_range', start_range)
if end_range_required:
_validate_not_none('end_range', end_range)
# Format based on whether end_range is present
request.headers = request.headers or {}
header_name = 'x-ms-source-range' if is_source else 'x-ms-range'
if end_range is not None:
request.headers[header_name] = 'bytes={0}-{1}'.format(
start_range, end_range)
elif start_range is not None:
request.headers[header_name] = 'bytes={0}-'.format(start_range)
# Content MD5 can only be provided for a complete range less than 4MB in size
if check_content_md5:
if start_range is None or end_range is None:
raise ValueError(_ERROR_START_END_NEEDED_FOR_MD5)
if end_range - start_range > 4 * 1024 * 1024:
raise ValueError(_ERROR_RANGE_TOO_LARGE_FOR_MD5)
request.headers['x-ms-range-get-content-md5'] = 'true'
def _validate_entity(entity, encrypt=None):
# Validate entity exists
_validate_not_none('entity', entity)
# Entity inherits from dict, so just validating dict is fine
if not isinstance(entity, dict):
raise TypeError(_ERROR_INVALID_ENTITY_TYPE)
# Validate partition key and row key are present
_validate_object_has_param('PartitionKey', entity)
_validate_object_has_param('RowKey', entity)
# Two properties are added during encryption. Validate sufficient space
max_properties = 255
if encrypt:
max_properties = max_properties - 2
# Validate there are not more than 255 properties including Timestamp
if (len(entity) > max_properties) or (len(entity) == max_properties
and 'Timestamp' not in entity):
raise ValueError(_ERROR_TOO_MANY_PROPERTIES)
# Validate the property names are not too long
for propname in entity:
if len(propname) > 255:
raise ValueError(_ERROR_PROPERTY_NAME_TOO_LONG)
def _update_entity(entity,
if_match,
encryption_required=False,
key_encryption_key=None,
encryption_resolver=None):
'''
Constructs an update entity request.
:param entity:
The entity to insert. Could be a dict or an entity object.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:param function(partition_key, row_key, property_name) encryption_resolver:
A function that takes in an entities partition key, row key, and property name and returns
a boolean that indicates whether that property should be encrypted.
'''
_validate_not_none('if_match', if_match)
_validate_entity(entity, key_encryption_key is not None)
_validate_encryption_required(encryption_required, key_encryption_key)
request = HTTPRequest()
request.method = 'PUT'
request.headers = {
_DEFAULT_CONTENT_TYPE_HEADER[0]: _DEFAULT_CONTENT_TYPE_HEADER[1],
_DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1],
'If-Match': _to_str(if_match),
}
if key_encryption_key:
entity = _encrypt_entity(entity, key_encryption_key,
encryption_resolver)
request.body = _get_request_body(_convert_entity_to_json(entity))
return request
def exists(self, queue_name, timeout=None):
'''
Returns a boolean indicating whether the queue exists.
:param str queue_name:
The name of queue to check for existence.
:param int timeout:
The server timeout, expressed in seconds.
:return: A boolean indicating whether the queue exists.
:rtype: bool
'''
_validate_not_none('queue_name', queue_name)
try:
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(queue_name)
request.query = {
'comp': 'metadata',
'timeout': _int_to_str(timeout),
}
self._perform_request(request, expected_errors=[_QUEUE_NOT_FOUND_ERROR_CODE])
return True
except AzureHttpError as ex:
_dont_fail_not_exist(ex)
return False
def set_queue_metadata(self, queue_name, metadata=None, timeout=None):
'''
Sets user-defined metadata on the specified queue. Metadata is
associated with the queue as name-value pairs.
:param str queue_name:
The name of an existing queue.
:param dict metadata:
A dict containing name-value pairs to associate with the
queue as metadata.
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
request.query = {
'comp': 'metadata',
'timeout': _int_to_str(timeout),
}
_add_metadata_headers(metadata, request)
self._perform_request(request)
def get_queue_metadata(self, queue_name, timeout=None):
'''
Retrieves user-defined metadata and queue properties on the specified
queue. Metadata is associated with the queue as name-value pairs.
:param str queue_name:
The name of an existing queue.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A dictionary representing the queue metadata with an
approximate_message_count int property on the dict estimating the
number of messages in the queue.
:rtype: dict(str, str)
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(queue_name)
request.query = {
'comp': 'metadata',
'timeout': _int_to_str(timeout),
}
return self._perform_request(request, _parse_metadata_and_message_count)
async def set_queue_metadata(self,
queue_name,
metadata=None,
timeout=None):
'''
Sets user-defined metadata on the specified queue. Metadata is
associated with the queue as name-value pairs.
:param str queue_name:
The name of an existing queue.
:param dict metadata:
A dict containing name-value pairs to associate with the
queue as metadata.
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
self._set_basic_headers(request)
request.query = {
'comp': 'metadata',
'timeout': _int_to_str(timeout),
}
_add_metadata_headers(metadata, request)
await self._perform_request(request)
def _validate_entity(entity, encrypt=None):
# Validate entity exists
_validate_not_none('entity', entity)
# Entity inherits from dict, so just validating dict is fine
if not isinstance(entity, dict):
raise TypeError(_ERROR_INVALID_ENTITY_TYPE)
# Validate partition key and row key are present
_validate_object_has_param('PartitionKey', entity)
_validate_object_has_param('RowKey', entity)
# Two properties are added during encryption. Validate sufficient space
max_properties = 255
if encrypt:
max_properties = max_properties - 2
# Validate there are not more than 255 properties including Timestamp
if (len(entity) > max_properties) or (len(entity) == max_properties and 'Timestamp' not in entity):
raise ValueError(_ERROR_TOO_MANY_PROPERTIES)
# Validate the property names are not too long
for propname in entity:
if len(propname) > 255:
raise ValueError(_ERROR_PROPERTY_NAME_TOO_LONG)
async def get_queue_metadata(self, queue_name, timeout=None):
'''
Retrieves user-defined metadata and queue properties on the specified
queue. Metadata is associated with the queue as name-value pairs.
:param str queue_name:
The name of an existing queue.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A dictionary representing the queue metadata with an
approximate_message_count int property on the dict estimating the
number of messages in the queue.
:rtype: dict(str, str)
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(queue_name)
request.query = {
'comp': 'metadata',
'timeout': _int_to_str(timeout),
}
return await self._perform_request(request,
_parse_metadata_and_message_count)
def _update_entity(entity, if_match, encryption_required=False,
key_encryption_key=None, encryption_resolver=None):
'''
Constructs an update entity request.
:param entity:
The entity to insert. Could be a dict or an entity object.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:param function(partition_key, row_key, property_name) encryption_resolver:
A function that takes in an entities partition key, row key, and property name and returns
a boolean that indicates whether that property should be encrypted.
'''
_validate_not_none('if_match', if_match)
_validate_entity(entity, key_encryption_key is not None)
_validate_encryption_required(encryption_required, key_encryption_key)
request = HTTPRequest()
request.method = 'PUT'
request.headers = {
_DEFAULT_CONTENT_TYPE_HEADER[0]: _DEFAULT_CONTENT_TYPE_HEADER[1],
_DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1],
'If-Match': _to_str(if_match),
}
if key_encryption_key:
entity = _encrypt_entity(entity, key_encryption_key, encryption_resolver)
request.body = _get_request_body(_convert_entity_to_json(entity))
return request
def _validate_and_format_range_headers(request, start_range, end_range, start_range_required=True,
end_range_required=True, check_content_md5=False, align_to_page=False):
# If end range is provided, start range must be provided
if start_range_required or end_range is not None:
_validate_not_none('start_range', start_range)
if end_range_required:
_validate_not_none('end_range', end_range)
# Page ranges must be 512 aligned
if align_to_page:
if start_range is not None and start_range % 512 != 0:
raise ValueError(_ERROR_PAGE_BLOB_START_ALIGNMENT)
if end_range is not None and end_range % 512 != 511:
raise ValueError(_ERROR_PAGE_BLOB_END_ALIGNMENT)
# Format based on whether end_range is present
request.headers = request.headers or {}
if end_range is not None:
request.headers['x-ms-range'] = 'bytes={0}-{1}'.format(start_range, end_range)
elif start_range is not None:
request.headers['x-ms-range'] = "bytes={0}-".format(start_range)
# Content MD5 can only be provided for a complete range less than 4MB in size
if check_content_md5:
if start_range is None or end_range is None:
raise ValueError(_ERROR_START_END_NEEDED_FOR_MD5)
if end_range - start_range > 4 * 1024 * 1024:
raise ValueError(_ERROR_RANGE_TOO_LARGE_FOR_MD5)
request.headers['x-ms-range-get-content-md5'] = 'true'
async def create_queue(self,
queue_name,
metadata=None,
fail_on_exist=False,
timeout=None):
'''
Creates a queue under the given account.
:param str queue_name:
The name of the queue to create. A queue name must be from 3 through
63 characters long and may only contain lowercase letters, numbers,
and the dash (-) character. The first and last letters in the queue
must be alphanumeric. The dash (-) character cannot be the first or
last character. Consecutive dash characters are not permitted in the
queue name.
:param metadata:
A dict containing name-value pairs to associate with the queue as
metadata. Note that metadata names preserve the case with which they
were created, but are case-insensitive when set or read.
:type metadata: dict(str, str)
:param bool fail_on_exist:
Specifies whether to throw an exception if the queue already exists.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A boolean indicating whether the queue was created. If fail_on_exist
was set to True, this will throw instead of returning false.
:rtype: bool
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
request.query = {'timeout': _int_to_str(timeout)}
self._set_basic_headers(request)
_add_metadata_headers(metadata, request)
def _return_request(request):
return request
if not fail_on_exist:
try:
response = await self._perform_request(request,
parser=_return_request)
if response.status == _HTTP_RESPONSE_NO_CONTENT:
return False
return True
except AzureHttpError as ex:
_dont_fail_on_exist(ex)
return False
else:
response = await self._perform_request(request,
parser=_return_request)
if response.status == _HTTP_RESPONSE_NO_CONTENT:
raise AzureConflictHttpError(
_ERROR_CONFLICT.format(response.message), response.status)
return True
def generate_account_shared_access_signature(self,
resource_types,
permission,
expiry,
start=None,
ip=None,
protocol=None):
'''
Generates a shared access signature for the queue service.
Use the returned signature with the sas_token parameter of QueueService.
:param ResourceTypes resource_types:
Specifies the resource types that are accessible with the account SAS.
:param AccountPermissions permission:
The permissions associated with the shared access signature. The
user is restricted to operations allowed by the permissions.
Required unless an id is given referencing a stored access policy
which contains this field. This field must be omitted if it has been
specified in an associated stored access policy.
:param expiry:
The time at which the shared access signature becomes invalid.
Required unless an id is given referencing a stored access policy
which contains this field. This field must be omitted if it has
been specified in an associated stored access policy. Azure will always
convert values to UTC. If a date is passed in without timezone info, it
is assumed to be UTC.
:type expiry: datetime or str
:param start:
The time at which the shared access signature becomes valid. If
omitted, start time for this call is assumed to be the time when the
storage service receives the request. Azure will always convert values
to UTC. If a date is passed in without timezone info, it is assumed to
be UTC.
:type start: datetime or str
:param str ip:
Specifies an IP address or a range of IP addresses from which to accept requests.
If the IP address from which the request originates does not match the IP address
or address range specified on the SAS token, the request is not authenticated.
For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS
restricts the request to those IP addresses.
:param str protocol:
Specifies the protocol permitted for a request made. The default value
is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values.
:return: A Shared Access Signature (sas) token.
:rtype: str
'''
_validate_not_none('self.account_name', self.account_name)
_validate_not_none('self.account_key', self.account_key)
sas = QueueSharedAccessSignature(self.account_name, self.account_key)
return sas.generate_account(Services.QUEUE,
resource_types,
permission,
expiry,
start=start,
ip=ip,
protocol=protocol)
async def get_messages(self,
queue_name,
num_messages=None,
visibility_timeout=None,
timeout=None):
'''
Retrieves one or more messages from the front of the queue.
When a message is retrieved from the queue, the response includes the message
content and a pop_receipt value, which is required to delete the message.
The message is not automatically deleted from the queue, but after it has
been retrieved, it is not visible to other clients for the time interval
specified by the visibility_timeout parameter.
If the key-encryption-key or resolver field is set on the local service object, the messages will be
decrypted before being returned.
:param str queue_name:
The name of the queue to get messages from.
:param int num_messages:
A nonzero integer value that specifies the number of
messages to retrieve from the queue, up to a maximum of 32. If
fewer are visible, the visible messages are returned. By default,
a single message is retrieved from the queue with this operation.
:param int visibility_timeout:
Specifies the new visibility timeout value, in seconds, relative
to server time. The new value must be larger than or equal to 1
second, and cannot be larger than 7 days. The visibility timeout of
a message can be set to a value later than the expiry time.
:param int timeout:
The server timeout, expressed in seconds.
:return: A :class:`~azure.storage.queue.models.QueueMessage` object representing the information passed.
:rtype: list(:class:`~azure.storage.queue.models.QueueMessage`)
'''
_validate_decryption_required(self.require_encryption,
self.key_encryption_key,
self.key_resolver_function)
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name, True)
request.query = {
'numofmessages': _to_str(num_messages),
'visibilitytimeout': _to_str(visibility_timeout),
'timeout': _int_to_str(timeout)
}
return await self._perform_request(
request, _convert_xml_to_queue_messages, [
self.decode_function, self.require_encryption,
self.key_encryption_key, self.key_resolver_function
])
def _decrypt_entity(entity, encrypted_properties_list, content_encryption_key,
entityIV, isJavaV1):
'''
Decrypts the specified entity using AES256 in CBC mode with 128 bit padding. Unwraps the CEK
using either the specified KEK or the key returned by the key_resolver. Properties
specified in the encrypted_properties_list, will be decrypted and decoded to utf-8 strings.
:param entity:
The entity being retrieved and decrypted. Could be a dict or an entity object.
:param list encrypted_properties_list:
The encrypted list of all the properties that are encrypted.
:param bytes[] content_encryption_key:
The key used internally to encrypt the entity. Extrated from the entity metadata.
:param bytes[] entityIV:
The intialization vector used to seed the encryption algorithm. Extracted from the
entity metadata.
:return: The decrypted entity
:rtype: Entity
'''
_validate_not_none('entity', entity)
decrypted_entity = deepcopy(entity)
try:
for property in entity.keys():
if property in encrypted_properties_list:
value = entity[property]
propertyIV = _generate_property_iv(entityIV,
entity['PartitionKey'],
entity['RowKey'], property,
isJavaV1)
cipher = _generate_AES_CBC_cipher(content_encryption_key,
propertyIV)
# Decrypt the property.
decryptor = cipher.decryptor()
decrypted_data = (decryptor.update(value.value) +
decryptor.finalize())
# Unpad the data.
unpadder = PKCS7(128).unpadder()
decrypted_data = (unpadder.update(decrypted_data) +
unpadder.finalize())
decrypted_data = decrypted_data.decode('utf-8')
decrypted_entity[property] = decrypted_data
decrypted_entity.pop('_ClientEncryptionMetadata1')
decrypted_entity.pop('_ClientEncryptionMetadata2')
return decrypted_entity
except:
raise AzureException(_ERROR_DECRYPTION_FAILURE)
def generate_shared_access_signature(self, services, resource_types,
permission, expiry, start=None,
ip=None, protocol=None):
'''
Generates a shared access signature for the account.
Use the returned signature with the sas_token parameter of the service
or to create a new account object.
:param Services services:
Specifies the services accessible with the account SAS. You can
combine values to provide access to more than one service.
:param ResourceTypes resource_types:
Specifies the resource types that are accessible with the account
SAS. You can combine values to provide access to more than one
resource type.
:param AccountPermissions permission:
The permissions associated with the shared access signature. The
user is restricted to operations allowed by the permissions.
Required unless an id is given referencing a stored access policy
which contains this field. This field must be omitted if it has been
specified in an associated stored access policy. You can combine
values to provide more than one permission.
:param expiry:
The time at which the shared access signature becomes invalid.
Required unless an id is given referencing a stored access policy
which contains this field. This field must be omitted if it has
been specified in an associated stored access policy. Azure will always
convert values to UTC. If a date is passed in without timezone info, it
is assumed to be UTC.
:type expiry: datetime or str
:param start:
The time at which the shared access signature becomes valid. If
omitted, start time for this call is assumed to be the time when the
storage service receives the request. Azure will always convert values
to UTC. If a date is passed in without timezone info, it is assumed to
be UTC.
:type start: datetime or str
:param str ip:
Specifies an IP address or a range of IP addresses from which to accept requests.
If the IP address from which the request originates does not match the IP address
or address range specified on the SAS token, the request is not authenticated.
For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS
restricts the request to those IP addresses.
:param str protocol:
Specifies the protocol permitted for a request made. Possible values are
both HTTPS and HTTP (https,http) or HTTPS only (https). The default value
is https,http. Note that HTTP only is not a permitted value.
'''
_validate_not_none('self.account_name', self.account_name)
_validate_not_none('self.account_key', self.account_key)
sas = SharedAccessSignature(self.account_name, self.account_key)
return sas.generate_account(services, resource_types, permission,
expiry, start=start, ip=ip, protocol=protocol)
async def peek_messages(self, queue_name, num_messages=None, timeout=None):
'''
Retrieves one or more messages from the front of the queue, but does
not alter the visibility of the message.
Only messages that are visible may be retrieved. When a message is retrieved
for the first time with a call to get_messages, its dequeue_count property
is set to 1. If it is not deleted and is subsequently retrieved again, the
dequeue_count property is incremented. The client may use this value to
determine how many times a message has been retrieved. Note that a call
to peek_messages does not increment the value of DequeueCount, but returns
this value for the client to read.
If the key-encryption-key or resolver field is set on the local service object, the messages will be
decrypted before being returned.
:param str queue_name:
The name of the queue to peek messages from.
:param int num_messages:
A nonzero integer value that specifies the number of
messages to peek from the queue, up to a maximum of 32. By default,
a single message is peeked from the queue with this operation.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A list of :class:`~azure.storage.queue.models.QueueMessage` objects. Note that
time_next_visible and pop_receipt will not be populated as peek does
not pop the message and can only retrieve already visible messages.
:rtype: list(:class:`~azure.storage.queue.models.QueueMessage`)
'''
_validate_decryption_required(self.require_encryption,
self.key_encryption_key,
self.key_resolver_function)
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(queue_name, True)
request.query = {
'peekonly': 'true',
'numofmessages': _to_str(num_messages),
'timeout': _int_to_str(timeout)
}
return await self._perform_request(
request, _convert_xml_to_queue_messages, [
self.decode_function, self.require_encryption,
self.key_encryption_key, self.key_resolver_function
])
def _encrypt_queue_message(message, key_encryption_key):
'''
Encrypts the given plain text message using AES256 in CBC mode with 128 bit padding.
Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek).
Returns a json-formatted string containing the encrypted message and the encryption metadata.
:param object message:
The plain text messge to be encrypted.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:return: A json-formatted string containing the encrypted message and the encryption metadata.
:rtype: str
'''
_validate_not_none('message', message)
_validate_not_none('key_encryption_key', key_encryption_key)
_validate_key_encryption_key_wrap(key_encryption_key)
# AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks
content_encryption_key = os.urandom(32)
initialization_vector = os.urandom(16)
# Queue encoding functions all return unicode strings, and encryption should
# operate on binary strings.
message = message.encode('utf-8')
cipher = _generate_AES_CBC_cipher(content_encryption_key,
initialization_vector)
# PKCS7 with 16 byte blocks ensures compatibility with AES.
padder = PKCS7(128).padder()
padded_data = padder.update(message) + padder.finalize()
# Encrypt the data.
encryptor = cipher.encryptor()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
# Build the dictionary structure.
queue_message = {
'EncryptedMessageContents':
_encode_base64(encrypted_data),
'EncryptionData':
_generate_encryption_data_dict(key_encryption_key,
content_encryption_key,
initialization_vector)
}
return dumps(queue_message)
def create_queue(self, queue_name, metadata=None, fail_on_exist=False, timeout=None):
'''
Creates a queue under the given account.
:param str queue_name:
The name of the queue to create. A queue name must be from 3 through
63 characters long and may only contain lowercase letters, numbers,
and the dash (-) character. The first and last letters in the queue
must be alphanumeric. The dash (-) character cannot be the first or
last character. Consecutive dash characters are not permitted in the
queue name.
:param metadata:
A dict containing name-value pairs to associate with the queue as
metadata. Note that metadata names preserve the case with which they
were created, but are case-insensitive when set or read.
:type metadata: dict(str, str)
:param bool fail_on_exist:
Specifies whether to throw an exception if the queue already exists.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A boolean indicating whether the queue was created. If fail_on_exist
was set to True, this will throw instead of returning false.
:rtype: bool
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
request.query = {'timeout': _int_to_str(timeout)}
_add_metadata_headers(metadata, request)
def _return_request(request):
return request
if not fail_on_exist:
try:
response = self._perform_request(request, parser=_return_request)
if response.status == _HTTP_RESPONSE_NO_CONTENT:
return False
return True
except AzureHttpError as ex:
_dont_fail_on_exist(ex)
return False
else:
response = self._perform_request(request, parser=_return_request)
if response.status == _HTTP_RESPONSE_NO_CONTENT:
raise AzureConflictHttpError(
_ERROR_CONFLICT.format(response.message), response.status)
return True
def _decrypt_entity(entity, encrypted_properties_list, content_encryption_key, entityIV, isJavaV1):
'''
Decrypts the specified entity using AES256 in CBC mode with 128 bit padding. Unwraps the CEK
using either the specified KEK or the key returned by the key_resolver. Properties
specified in the encrypted_properties_list, will be decrypted and decoded to utf-8 strings.
:param entity:
The entity being retrieved and decrypted. Could be a dict or an entity object.
:param list encrypted_properties_list:
The encrypted list of all the properties that are encrypted.
:param bytes[] content_encryption_key:
The key used internally to encrypt the entity. Extrated from the entity metadata.
:param bytes[] entityIV:
The intialization vector used to seed the encryption algorithm. Extracted from the
entity metadata.
:return: The decrypted entity
:rtype: Entity
'''
_validate_not_none('entity', entity)
decrypted_entity = deepcopy(entity)
try:
for property in entity.keys():
if property in encrypted_properties_list:
value = entity[property]
propertyIV = _generate_property_iv(entityIV,
entity['PartitionKey'], entity['RowKey'],
property, isJavaV1)
cipher = _generate_AES_CBC_cipher(content_encryption_key,
propertyIV)
# Decrypt the property.
decryptor = cipher.decryptor()
decrypted_data = (decryptor.update(value.value) + decryptor.finalize())
# Unpad the data.
unpadder = PKCS7(128).unpadder()
decrypted_data = (unpadder.update(decrypted_data) + unpadder.finalize())
decrypted_data = decrypted_data.decode('utf-8')
decrypted_entity[property] = decrypted_data
decrypted_entity.pop('_ClientEncryptionMetadata1')
decrypted_entity.pop('_ClientEncryptionMetadata2')
return decrypted_entity
except:
raise AzureException(_ERROR_DECRYPTION_FAILURE)
def get_messages(self, queue_name, num_messages=None,
visibility_timeout=None, timeout=None):
'''
Retrieves one or more messages from the front of the queue.
When a message is retrieved from the queue, the response includes the message
content and a pop_receipt value, which is required to delete the message.
The message is not automatically deleted from the queue, but after it has
been retrieved, it is not visible to other clients for the time interval
specified by the visibility_timeout parameter.
If the key-encryption-key or resolver field is set on the local service object, the messages will be
decrypted before being returned.
:param str queue_name:
The name of the queue to get messages from.
:param int num_messages:
A nonzero integer value that specifies the number of
messages to retrieve from the queue, up to a maximum of 32. If
fewer are visible, the visible messages are returned. By default,
a single message is retrieved from the queue with this operation.
:param int visibility_timeout:
Specifies the new visibility timeout value, in seconds, relative
to server time. The new value must be larger than or equal to 1
second, and cannot be larger than 7 days. The visibility timeout of
a message can be set to a value later than the expiry time.
:param int timeout:
The server timeout, expressed in seconds.
:return: A :class:`~azure.storage.queue.models.QueueMessage` object representing the information passed.
:rtype: list(:class:`~azure.storage.queue.models.QueueMessage`)
'''
_validate_decryption_required(self.require_encryption, self.key_encryption_key,
self.key_resolver_function)
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name, True)
request.query = {
'numofmessages': _to_str(num_messages),
'visibilitytimeout': _to_str(visibility_timeout),
'timeout': _int_to_str(timeout)
}
return self._perform_request(request, _convert_xml_to_queue_messages,
[self.decode_function, self.require_encryption,
self.key_encryption_key, self.key_resolver_function])
def peek_messages(self, queue_name, num_messages=None, timeout=None):
'''
Retrieves one or more messages from the front of the queue, but does
not alter the visibility of the message.
Only messages that are visible may be retrieved. When a message is retrieved
for the first time with a call to get_messages, its dequeue_count property
is set to 1. If it is not deleted and is subsequently retrieved again, the
dequeue_count property is incremented. The client may use this value to
determine how many times a message has been retrieved. Note that a call
to peek_messages does not increment the value of DequeueCount, but returns
this value for the client to read.
If the key-encryption-key or resolver field is set on the local service object, the messages will be
decrypted before being returned.
:param str queue_name:
The name of the queue to peek messages from.
:param int num_messages:
A nonzero integer value that specifies the number of
messages to peek from the queue, up to a maximum of 32. By default,
a single message is peeked from the queue with this operation.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A list of :class:`~azure.storage.queue.models.QueueMessage` objects. Note that
time_next_visible and pop_receipt will not be populated as peek does
not pop the message and can only retrieve already visible messages.
:rtype: list(:class:`~azure.storage.queue.models.QueueMessage`)
'''
_validate_decryption_required(self.require_encryption, self.key_encryption_key,
self.key_resolver_function)
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(queue_name, True)
request.query = {
'peekonly': 'true',
'numofmessages': _to_str(num_messages),
'timeout': _int_to_str(timeout)
}
return self._perform_request(request, _convert_xml_to_queue_messages,
[self.decode_function, self.require_encryption,
self.key_encryption_key, self.key_resolver_function])
def clear_messages(self, queue_name, timeout=None):
'''
Deletes all messages from the specified queue.
:param str queue_name:
The name of the queue whose messages to clear.
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'DELETE'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name, True)
request.query = {'timeout': _int_to_str(timeout)}
self._perform_request(request)
def clear_messages(self, queue_name, timeout=None):
'''
Deletes all messages from the specified queue.
:param str queue_name:
The name of the queue whose messages to clear.
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'DELETE'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name, True)
request.query = {'timeout': _int_to_str(timeout)}
self._perform_request(request)
def generate_account_shared_access_signature(self, resource_types, permission,
expiry, start=None, ip=None, protocol=None):
'''
Generates a shared access signature for the queue service.
Use the returned signature with the sas_token parameter of QueueService.
:param ResourceTypes resource_types:
Specifies the resource types that are accessible with the account SAS.
:param AccountPermissions permission:
The permissions associated with the shared access signature. The
user is restricted to operations allowed by the permissions.
Required unless an id is given referencing a stored access policy
which contains this field. This field must be omitted if it has been
specified in an associated stored access policy.
:param expiry:
The time at which the shared access signature becomes invalid.
Required unless an id is given referencing a stored access policy
which contains this field. This field must be omitted if it has
been specified in an associated stored access policy. Azure will always
convert values to UTC. If a date is passed in without timezone info, it
is assumed to be UTC.
:type expiry: datetime or str
:param start:
The time at which the shared access signature becomes valid. If
omitted, start time for this call is assumed to be the time when the
storage service receives the request. Azure will always convert values
to UTC. If a date is passed in without timezone info, it is assumed to
be UTC.
:type start: datetime or str
:param str ip:
Specifies an IP address or a range of IP addresses from which to accept requests.
If the IP address from which the request originates does not match the IP address
or address range specified on the SAS token, the request is not authenticated.
For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS
restricts the request to those IP addresses.
:param str protocol:
Specifies the protocol permitted for a request made. The default value
is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values.
:return: A Shared Access Signature (sas) token.
:rtype: str
'''
_validate_not_none('self.account_name', self.account_name)
_validate_not_none('self.account_key', self.account_key)
sas = QueueSharedAccessSignature(self.account_name, self.account_key)
return sas.generate_account(Services.QUEUE, resource_types, permission,
expiry, start=start, ip=ip, protocol=protocol)
def get_block_list(self, container_name, blob_name, snapshot=None,
block_list_type=None, lease_id=None, timeout=None):
'''
Retrieves the list of blocks that have been uploaded as part of a
block blob. There are two block lists maintained for a blob:
Committed Block List:
The list of blocks that have been successfully committed to a
given blob with Put Block List.
Uncommitted Block List:
The list of blocks that have been uploaded for a blob using
Put Block, but that have not yet been committed. These blocks
are stored in Azure in association with a blob, but do not yet
form part of the blob.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of existing blob.
:param str snapshot:
Datetime to determine the time to retrieve the blocks.
:param str block_list_type:
Specifies whether to return the list of committed blocks, the list
of uncommitted blocks, or both lists together. Valid values are:
committed, uncommitted, or all.
:param str lease_id:
Required if the blob has an active lease.
:param int timeout:
The timeout parameter is expressed in seconds.
:return: list committed and/or uncommitted blocks for Block Blob
:rtype: :class:`~azure.storage.blob.models.BlobBlockList`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'blocklist',
'snapshot': _to_str(snapshot),
'blocklisttype': _to_str(block_list_type),
'timeout': _int_to_str(timeout),
}
request.headers = {'x-ms-lease-id': _to_str(lease_id)}
return self._perform_request(request, _convert_xml_to_block_list)
def get_block_list(self, container_name, blob_name, snapshot=None,
block_list_type=None, lease_id=None, timeout=None):
'''
Retrieves the list of blocks that have been uploaded as part of a
block blob. There are two block lists maintained for a blob:
Committed Block List:
The list of blocks that have been successfully committed to a
given blob with Put Block List.
Uncommitted Block List:
The list of blocks that have been uploaded for a blob using
Put Block, but that have not yet been committed. These blocks
are stored in Azure in association with a blob, but do not yet
form part of the blob.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of existing blob.
:param str snapshot:
Datetime to determine the time to retrieve the blocks.
:param str block_list_type:
Specifies whether to return the list of committed blocks, the list
of uncommitted blocks, or both lists together. Valid values are:
committed, uncommitted, or all.
:param str lease_id:
Required if the blob has an active lease.
:param int timeout:
The timeout parameter is expressed in seconds.
:return: list committed and/or uncommitted blocks for Block Blob
:rtype: :class:`~azure.storage.blob.models.BlobBlockList`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'blocklist',
'snapshot': _to_str(snapshot),
'blocklisttype': _to_str(block_list_type),
'timeout': _int_to_str(timeout),
}
request.headers = {'x-ms-lease-id': _to_str(lease_id)}
return self._perform_request(request, _convert_xml_to_block_list)
def _merge_entity(entity, if_match, require_encryption=False, key_encryption_key=None):
'''
Constructs a merge entity request.
'''
_validate_not_none('if_match', if_match)
_validate_entity(entity)
_validate_encryption_unsupported(require_encryption, key_encryption_key)
request = HTTPRequest()
request.method = 'MERGE'
request.headers = {
_DEFAULT_CONTENT_TYPE_HEADER[0]: _DEFAULT_CONTENT_TYPE_HEADER[1],
_DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1],
'If-Match': _to_str(if_match)
}
request.body = _get_request_body(_convert_entity_to_json(entity))
return request
def _encrypt_blob(blob, key_encryption_key):
'''
Encrypts the given blob using AES256 in CBC mode with 128 bit padding.
Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek).
Returns a json-formatted string containing the encryption metadata. This method should
only be used when a blob is small enough for single shot upload. Encrypting larger blobs
is done as a part of the _upload_blob_chunks method.
:param bytes blob:
The blob to be encrypted.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:return: A tuple of json-formatted string containing the encryption metadata and the encrypted blob data.
:rtype: (str, bytes)
'''
_validate_not_none('blob', blob)
_validate_not_none('key_encryption_key', key_encryption_key)
_validate_key_encryption_key_wrap(key_encryption_key)
# AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks
content_encryption_key = urandom(32)
initialization_vector = urandom(16)
cipher = _generate_AES_CBC_cipher(content_encryption_key,
initialization_vector)
# PKCS7 with 16 byte blocks ensures compatibility with AES.
padder = PKCS7(128).padder()
padded_data = padder.update(blob) + padder.finalize()
# Encrypt the data.
encryptor = cipher.encryptor()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
encryption_data = _generate_encryption_data_dict(key_encryption_key,
content_encryption_key,
initialization_vector)
encryption_data['EncryptionMode'] = 'FullBlob'
return dumps(encryption_data), encrypted_data
async def set_queue_acl(self,
queue_name,
signed_identifiers=None,
timeout=None):
'''
Sets stored access policies for the queue that may be used with Shared
Access Signatures.
When you set permissions for a queue, the existing permissions are replaced.
To update the queue's permissions, call :func:`~get_queue_acl` to fetch
all access policies associated with the queue, modify the access policy
that you wish to change, and then call this function with the complete
set of data to perform the update.
When you establish a stored access policy on a queue, it may take up to
30 seconds to take effect. During this interval, a shared access signature
that is associated with the stored access policy will throw an
:class:`AzureHttpError` until the access policy becomes active.
:param str queue_name:
The name of an existing queue.
:param signed_identifiers:
A dictionary of access policies to associate with the queue. The
dictionary may contain up to 5 elements. An empty dictionary
will clear the access policies set on the service.
:type signed_identifiers: dict(str, :class:`~azure.storage.common.models.AccessPolicy`)
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
_validate_access_policies(signed_identifiers)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
self._set_basic_headers(request)
request.query = {
'comp': 'acl',
'timeout': _int_to_str(timeout),
}
request.body = _get_request_body(
_convert_signed_identifiers_to_xml(signed_identifiers))
await self._perform_request(request)
async def delete_queue(self,
queue_name,
fail_not_exist=False,
timeout=None):
'''
Deletes the specified queue and any messages it contains.
When a queue is successfully deleted, it is immediately marked for deletion
and is no longer accessible to clients. The queue is later removed from
the Queue service during garbage collection.
Note that deleting a queue is likely to take at least 40 seconds to complete.
If an operation is attempted against the queue while it was being deleted,
an :class:`AzureConflictHttpError` will be thrown.
:param str queue_name:
The name of the queue to delete.
:param bool fail_not_exist:
Specifies whether to throw an exception if the queue doesn't exist.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A boolean indicating whether the queue was deleted. If fail_not_exist
was set to True, this will throw instead of returning false.
:rtype: bool
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'DELETE'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
request.query = {'timeout': _int_to_str(timeout)}
self._set_basic_headers(request)
if not fail_not_exist:
try:
await self._perform_request(request)
return True
except AzureHttpError as ex:
_dont_fail_not_exist(ex)
return False
else:
await self._perform_request(request)
return True
def _encrypt_blob(blob, key_encryption_key):
'''
Encrypts the given blob using AES256 in CBC mode with 128 bit padding.
Wraps the generated content-encryption-key using the user-provided key-encryption-key (kek).
Returns a json-formatted string containing the encryption metadata. This method should
only be used when a blob is small enough for single shot upload. Encrypting larger blobs
is done as a part of the _upload_blob_chunks method.
:param bytes blob:
The blob to be encrypted.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:return: A tuple of json-formatted string containing the encryption metadata and the encrypted blob data.
:rtype: (str, bytes)
'''
_validate_not_none('blob', blob)
_validate_not_none('key_encryption_key', key_encryption_key)
_validate_key_encryption_key_wrap(key_encryption_key)
# AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks
content_encryption_key = urandom(32)
initialization_vector = urandom(16)
cipher = _generate_AES_CBC_cipher(content_encryption_key, initialization_vector)
# PKCS7 with 16 byte blocks ensures compatibility with AES.
padder = PKCS7(128).padder()
padded_data = padder.update(blob) + padder.finalize()
# Encrypt the data.
encryptor = cipher.encryptor()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
encryption_data = _generate_encryption_data_dict(key_encryption_key, content_encryption_key,
initialization_vector)
encryption_data['EncryptionMode'] = 'FullBlob'
return dumps(encryption_data), encrypted_data
def _merge_entity(entity,
if_match,
require_encryption=False,
key_encryption_key=None):
'''
Constructs a merge entity request.
'''
_validate_not_none('if_match', if_match)
_validate_entity(entity)
_validate_encryption_unsupported(require_encryption, key_encryption_key)
request = HTTPRequest()
request.method = 'MERGE'
request.headers = {
_DEFAULT_CONTENT_TYPE_HEADER[0]: _DEFAULT_CONTENT_TYPE_HEADER[1],
_DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1],
'If-Match': _to_str(if_match)
}
request.body = _get_request_body(_convert_entity_to_json(entity))
return request
def _decrypt(message, encryption_data, key_encryption_key=None, resolver=None):
'''
Decrypts the given ciphertext using AES256 in CBC mode with 128 bit padding.
Unwraps the content-encryption-key using the user-provided or resolved key-encryption-key (kek). Returns the original plaintex.
:param str message:
The ciphertext to be decrypted.
:param _EncryptionData encryption_data:
The metadata associated with this ciphertext.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
unwrap_key(key, algorithm)--returns the unwrapped form of the specified symmetric key using the string-specified algorithm.
get_kid()--returns a string key id for this key-encryption-key.
:param function resolver(kid):
The user-provided key resolver. Uses the kid string to return a key-encryption-key implementing the interface defined above.
:return: The decrypted plaintext.
:rtype: str
'''
_validate_not_none('message', message)
content_encryption_key = _validate_and_unwrap_cek(encryption_data,
key_encryption_key,
resolver)
if not (_EncryptionAlgorithm.AES_CBC_256
== encryption_data.encryption_agent.encryption_algorithm):
raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM)
cipher = _generate_AES_CBC_cipher(content_encryption_key,
encryption_data.content_encryption_IV)
# decrypt data
decrypted_data = message
decryptor = cipher.decryptor()
decrypted_data = (decryptor.update(decrypted_data) + decryptor.finalize())
# unpad data
unpadder = PKCS7(128).unpadder()
decrypted_data = (unpadder.update(decrypted_data) + unpadder.finalize())
return decrypted_data
def set_standard_blob_tier(
self, container_name, blob_name, standard_blob_tier, timeout=None):
'''
Sets the block blob tiers on the blob. This API is only supported for block blobs on standard storage accounts.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to update.
:param StandardBlobTier standard_blob_tier:
A standard blob tier value to set the blob to. For this version of the library,
this is only applicable to block blobs on standard storage accounts.
:param int timeout:
The timeout parameter is expressed in seconds. This method may make
multiple calls to the Azure service and the timeout will apply to
each call individually.
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('standard_blob_tier', standard_blob_tier)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'tier',
'timeout': _int_to_str(timeout),
}
request.headers = {
'x-ms-access-tier': _to_str(standard_blob_tier)
}
self._perform_request(request)
def set_standard_blob_tier(
self, container_name, blob_name, standard_blob_tier, timeout=None):
'''
Sets the block blob tiers on the blob. This API is only supported for block blobs on standard storage accounts.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to update.
:param StandardBlobTier standard_blob_tier:
A standard blob tier value to set the blob to. For this version of the library,
this is only applicable to block blobs on standard storage accounts.
:param int timeout:
The timeout parameter is expressed in seconds. This method may make
multiple calls to the Azure service and the timeout will apply to
each call individually.
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('standard_blob_tier', standard_blob_tier)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'tier',
'timeout': _int_to_str(timeout),
}
request.headers = {
'x-ms-access-tier': _to_str(standard_blob_tier)
}
self._perform_request(request)
def set_queue_acl(self, queue_name, signed_identifiers=None, timeout=None):
'''
Sets stored access policies for the queue that may be used with Shared
Access Signatures.
When you set permissions for a queue, the existing permissions are replaced.
To update the queue's permissions, call :func:`~get_queue_acl` to fetch
all access policies associated with the queue, modify the access policy
that you wish to change, and then call this function with the complete
set of data to perform the update.
When you establish a stored access policy on a queue, it may take up to
30 seconds to take effect. During this interval, a shared access signature
that is associated with the stored access policy will throw an
:class:`AzureHttpError` until the access policy becomes active.
:param str queue_name:
The name of an existing queue.
:param signed_identifiers:
A dictionary of access policies to associate with the queue. The
dictionary may contain up to 5 elements. An empty dictionary
will clear the access policies set on the service.
:type signed_identifiers: dict(str, :class:`~azure.storage.common.models.AccessPolicy`)
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
_validate_access_policies(signed_identifiers)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
request.query = {
'comp': 'acl',
'timeout': _int_to_str(timeout),
}
request.body = _get_request_body(
_convert_signed_identifiers_to_xml(signed_identifiers))
self._perform_request(request)
def delete_queue(self, queue_name, fail_not_exist=False, timeout=None):
'''
Deletes the specified queue and any messages it contains.
When a queue is successfully deleted, it is immediately marked for deletion
and is no longer accessible to clients. The queue is later removed from
the Queue service during garbage collection.
Note that deleting a queue is likely to take at least 40 seconds to complete.
If an operation is attempted against the queue while it was being deleted,
an :class:`AzureConflictHttpError` will be thrown.
:param str queue_name:
The name of the queue to delete.
:param bool fail_not_exist:
Specifies whether to throw an exception if the queue doesn't exist.
:param int timeout:
The server timeout, expressed in seconds.
:return:
A boolean indicating whether the queue was deleted. If fail_not_exist
was set to True, this will throw instead of returning false.
:rtype: bool
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'DELETE'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name)
request.query = {'timeout': _int_to_str(timeout)}
if not fail_not_exist:
try:
self._perform_request(request)
return True
except AzureHttpError as ex:
_dont_fail_not_exist(ex)
return False
else:
self._perform_request(request)
return True
def get_queue_acl(self, queue_name, timeout=None):
'''
Returns details about any stored access policies specified on the
queue that may be used with Shared Access Signatures.
:param str queue_name:
The name of an existing queue.
:param int timeout:
The server timeout, expressed in seconds.
:return: A dictionary of access policies associated with the queue.
:rtype: dict(str, :class:`~azure.storage.common.models.AccessPolicy`)
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(queue_name)
request.query = {
'comp': 'acl',
'timeout': _int_to_str(timeout),
}
return self._perform_request(request, _convert_xml_to_signed_identifiers)
def get_queue_acl(self, queue_name, timeout=None):
'''
Returns details about any stored access policies specified on the
queue that may be used with Shared Access Signatures.
:param str queue_name:
The name of an existing queue.
:param int timeout:
The server timeout, expressed in seconds.
:return: A dictionary of access policies associated with the queue.
:rtype: dict(str, :class:`~azure.storage.common.models.AccessPolicy`)
'''
_validate_not_none('queue_name', queue_name)
request = HTTPRequest()
request.method = 'GET'
request.host_locations = self._get_host_locations(secondary=True)
request.path = _get_path(queue_name)
request.query = {
'comp': 'acl',
'timeout': _int_to_str(timeout),
}
return self._perform_request(request, _convert_xml_to_signed_identifiers)
def _validate_and_format_range_headers(request,
start_range,
end_range,
start_range_required=True,
end_range_required=True,
check_content_md5=False,
align_to_page=False,
range_header_name='x-ms-range'):
# If end range is provided, start range must be provided
if start_range_required or end_range is not None:
_validate_not_none('start_range', start_range)
if end_range_required:
_validate_not_none('end_range', end_range)
# Page ranges must be 512 aligned
if align_to_page:
if start_range is not None and start_range % 512 != 0:
raise ValueError(_ERROR_PAGE_BLOB_START_ALIGNMENT)
if end_range is not None and end_range % 512 != 511:
raise ValueError(_ERROR_PAGE_BLOB_END_ALIGNMENT)
# Format based on whether end_range is present
request.headers = request.headers or {}
if end_range is not None:
request.headers[range_header_name] = 'bytes={0}-{1}'.format(
start_range, end_range)
elif start_range is not None:
request.headers[range_header_name] = "bytes={0}-".format(start_range)
# Content MD5 can only be provided for a complete range less than 4MB in size
if check_content_md5:
if start_range is None or end_range is None:
raise ValueError(_ERROR_START_END_NEEDED_FOR_MD5)
if end_range - start_range > 4 * 1024 * 1024:
raise ValueError(_ERROR_RANGE_TOO_LARGE_FOR_MD5)
request.headers['x-ms-range-get-content-md5'] = 'true'
def append_blob_from_path(
self, container_name, blob_name, file_path, validate_content=False,
maxsize_condition=None, progress_callback=None, lease_id=None, timeout=None):
'''
Appends to the content of an existing blob from a file path, with automatic
chunking and progress notifications.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to create or update.
:param str file_path:
Path of the file to upload as the blob content.
:param bool validate_content:
If true, calculates an MD5 hash for each chunk of the blob. The storage
service checks the hash of the content that has arrived with the hash
that was sent. This is primarily valuable for detecting bitflips on
the wire if using http instead of https as https (the default) will
already validate. Note that this MD5 hash is not stored with the
blob.
:param int maxsize_condition:
Optional conditional header. The max length in bytes permitted for
the append blob. If the Append Block operation would cause the blob
to exceed that limit or if the blob size is already greater than the
value specified in this header, the request will fail with
MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed).
:param progress_callback:
Callback for progress with signature function(current, total) where
current is the number of bytes transfered so far, and total is the
size of the blob, or None if the total size is unknown.
:type progress_callback: func(current, total)
:param str lease_id:
Required if the blob has an active lease.
:param int timeout:
The timeout parameter is expressed in seconds. This method may make
multiple calls to the Azure service and the timeout will apply to
each call individually.
:return: ETag and last modified properties for the Append Blob
:rtype: :class:`~azure.storage.blob.models.ResourceProperties`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('file_path', file_path)
_validate_encryption_unsupported(self.require_encryption, self.key_encryption_key)
count = path.getsize(file_path)
with open(file_path, 'rb') as stream:
return self.append_blob_from_stream(
container_name,
blob_name,
stream,
count=count,
validate_content=validate_content,
maxsize_condition=maxsize_condition,
progress_callback=progress_callback,
lease_id=lease_id,
timeout=timeout)
def append_blob_from_path(
self, container_name, blob_name, file_path, validate_content=False,
maxsize_condition=None, progress_callback=None, lease_id=None, timeout=None):
'''
Appends to the content of an existing blob from a file path, with automatic
chunking and progress notifications.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to create or update.
:param str file_path:
Path of the file to upload as the blob content.
:param bool validate_content:
If true, calculates an MD5 hash for each chunk of the blob. The storage
service checks the hash of the content that has arrived with the hash
that was sent. This is primarily valuable for detecting bitflips on
the wire if using http instead of https as https (the default) will
already validate. Note that this MD5 hash is not stored with the
blob.
:param int maxsize_condition:
Optional conditional header. The max length in bytes permitted for
the append blob. If the Append Block operation would cause the blob
to exceed that limit or if the blob size is already greater than the
value specified in this header, the request will fail with
MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed).
:param progress_callback:
Callback for progress with signature function(current, total) where
current is the number of bytes transfered so far, and total is the
size of the blob, or None if the total size is unknown.
:type progress_callback: func(current, total)
:param str lease_id:
Required if the blob has an active lease.
:param int timeout:
The timeout parameter is expressed in seconds. This method may make
multiple calls to the Azure service and the timeout will apply to
each call individually.
:return: ETag and last modified properties for the Append Blob
:rtype: :class:`~azure.storage.blob.models.ResourceProperties`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('file_path', file_path)
_validate_encryption_unsupported(self.require_encryption, self.key_encryption_key)
count = path.getsize(file_path)
with open(file_path, 'rb') as stream:
return self.append_blob_from_stream(
container_name,
blob_name,
stream,
count=count,
validate_content=validate_content,
maxsize_condition=maxsize_condition,
progress_callback=progress_callback,
lease_id=lease_id,
timeout=timeout)
def _get_entity(partition_key, row_key, select, accept):
'''
Constructs a get entity request.
'''
_validate_not_none('partition_key', partition_key)
_validate_not_none('row_key', row_key)
_validate_not_none('accept', accept)
request = HTTPRequest()
request.method = 'GET'
request.headers = {'Accept': _to_str(accept)}
request.query = {'$select': _to_str(select)}
return request
def _get_entity(partition_key, row_key, select, accept):
'''
Constructs a get entity request.
'''
_validate_not_none('partition_key', partition_key)
_validate_not_none('row_key', row_key)
_validate_not_none('accept', accept)
request = HTTPRequest()
request.method = 'GET'
request.headers = {'Accept': _to_str(accept)}
request.query = {'$select': _to_str(select)}
return request
def _put_block(self, container_name, blob_name, block, block_id,
validate_content=False, lease_id=None, timeout=None):
'''
See put_block for more details. This helper method
allows for encryption or other such special behavior because
it is safely handled by the library. These behaviors are
prohibited in the public version of this function.
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('block', block)
_validate_not_none('block_id', block_id)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'block',
'blockid': _encode_base64(_to_str(block_id)),
'timeout': _int_to_str(timeout),
}
request.headers = {
'x-ms-lease-id': _to_str(lease_id)
}
request.body = _get_data_bytes_or_stream_only('block', block)
if hasattr(request.body, 'read'):
if _len_plus(request.body) is None:
try:
data = b''
for chunk in iter(lambda: request.body.read(4096), b""):
data += chunk
request.body = data
except AttributeError:
raise ValueError(_ERROR_VALUE_SHOULD_BE_STREAM.format('request.body'))
if validate_content:
computed_md5 = _get_content_md5(request.body)
request.headers['Content-MD5'] = _to_str(computed_md5)
self._perform_request(request)
def _put_block(self, container_name, blob_name, block, block_id,
validate_content=False, lease_id=None, timeout=None):
'''
See put_block for more details. This helper method
allows for encryption or other such special behavior because
it is safely handled by the library. These behaviors are
prohibited in the public version of this function.
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('block', block)
_validate_not_none('block_id', block_id)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'block',
'blockid': _encode_base64(_to_str(block_id)),
'timeout': _int_to_str(timeout),
}
request.headers = {
'x-ms-lease-id': _to_str(lease_id)
}
request.body = _get_data_bytes_or_stream_only('block', block)
if hasattr(request.body, 'read'):
if _len_plus(request.body) is None:
try:
data = b''
for chunk in iter(lambda: request.body.read(4096), b""):
data += chunk
request.body = data
except AttributeError:
raise ValueError(_ERROR_VALUE_SHOULD_BE_STREAM.format('request.body'))
if validate_content:
computed_md5 = _get_content_md5(request.body)
request.headers['Content-MD5'] = _to_str(computed_md5)
self._perform_request(request)
def _delete_entity(partition_key, row_key, if_match):
'''
Constructs a delete entity request.
'''
_validate_not_none('if_match', if_match)
_validate_not_none('partition_key', partition_key)
_validate_not_none('row_key', row_key)
request = HTTPRequest()
request.method = 'DELETE'
request.headers = {
_DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1],
'If-Match': _to_str(if_match)
}
return request
def _delete_entity(partition_key, row_key, if_match):
'''
Constructs a delete entity request.
'''
_validate_not_none('if_match', if_match)
_validate_not_none('partition_key', partition_key)
_validate_not_none('row_key', row_key)
request = HTTPRequest()
request.method = 'DELETE'
request.headers = {
_DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1],
'If-Match': _to_str(if_match)
}
return request
def _put_block_list(
self, container_name, blob_name, block_list, content_settings=None,
metadata=None, validate_content=False, lease_id=None, if_modified_since=None,
if_unmodified_since=None, if_match=None, if_none_match=None,
timeout=None, encryption_data=None):
'''
See put_block_list for more details. This helper method
allows for encryption or other such special behavior because
it is safely handled by the library. These behaviors are
prohibited in the public version of this function.
:param str encryption_data:
A JSON formatted string containing the encryption metadata generated for this
blob if it was encrypted all at once upon upload. This should only be passed
in by internal methods.
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('block_list', block_list)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'blocklist',
'timeout': _int_to_str(timeout),
}
request.headers = {
'x-ms-lease-id': _to_str(lease_id),
'If-Modified-Since': _datetime_to_utc_string(if_modified_since),
'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since),
'If-Match': _to_str(if_match),
'If-None-Match': _to_str(if_none_match),
}
_add_metadata_headers(metadata, request)
if content_settings is not None:
request.headers.update(content_settings._to_headers())
request.body = _get_request_body(
_convert_block_list_to_xml(block_list))
if validate_content:
computed_md5 = _get_content_md5(request.body)
request.headers['Content-MD5'] = _to_str(computed_md5)
if encryption_data is not None:
request.headers['x-ms-meta-encryptiondata'] = encryption_data
return self._perform_request(request, _parse_base_properties)
def _put_block_list(
self, container_name, blob_name, block_list, content_settings=None,
metadata=None, validate_content=False, lease_id=None, if_modified_since=None,
if_unmodified_since=None, if_match=None, if_none_match=None,
timeout=None, encryption_data=None):
'''
See put_block_list for more details. This helper method
allows for encryption or other such special behavior because
it is safely handled by the library. These behaviors are
prohibited in the public version of this function.
:param str encryption_data:
A JSON formatted string containing the encryption metadata generated for this
blob if it was encrypted all at once upon upload. This should only be passed
in by internal methods.
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('block_list', block_list)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {
'comp': 'blocklist',
'timeout': _int_to_str(timeout),
}
request.headers = {
'x-ms-lease-id': _to_str(lease_id),
'If-Modified-Since': _datetime_to_utc_string(if_modified_since),
'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since),
'If-Match': _to_str(if_match),
'If-None-Match': _to_str(if_none_match),
}
_add_metadata_headers(metadata, request)
if content_settings is not None:
request.headers.update(content_settings._to_headers())
request.body = _get_request_body(
_convert_block_list_to_xml(block_list))
if validate_content:
computed_md5 = _get_content_md5(request.body)
request.headers['Content-MD5'] = _to_str(computed_md5)
if encryption_data is not None:
request.headers['x-ms-meta-encryptiondata'] = encryption_data
return self._perform_request(request, _parse_base_properties)
async def delete_message(self,
queue_name,
message_id,
pop_receipt,
timeout=None):
'''
Deletes the specified message.
Normally after a client retrieves a message with the get_messages operation,
the client is expected to process and delete the message. To delete the
message, you must have two items of data: id and pop_receipt. The
id is returned from the previous get_messages operation. The
pop_receipt is returned from the most recent :func:`~get_messages` or
:func:`~update_message` operation. In order for the delete_message operation
to succeed, the pop_receipt specified on the request must match the
pop_receipt returned from the :func:`~get_messages` or :func:`~update_message`
operation.
:param str queue_name:
The name of the queue from which to delete the message.
:param str message_id:
The message id identifying the message to delete.
:param str pop_receipt:
A valid pop receipt value returned from an earlier call
to the :func:`~get_messages` or :func:`~update_message`.
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
_validate_not_none('message_id', message_id)
_validate_not_none('pop_receipt', pop_receipt)
request = HTTPRequest()
request.method = 'DELETE'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name, True, message_id)
self._set_basic_headers(request)
request.query = {
'popreceipt': _to_str(pop_receipt),
'timeout': _int_to_str(timeout)
}
await self._perform_request(request)
def delete_message(self, queue_name, message_id, pop_receipt, timeout=None):
'''
Deletes the specified message.
Normally after a client retrieves a message with the get_messages operation,
the client is expected to process and delete the message. To delete the
message, you must have two items of data: id and pop_receipt. The
id is returned from the previous get_messages operation. The
pop_receipt is returned from the most recent :func:`~get_messages` or
:func:`~update_message` operation. In order for the delete_message operation
to succeed, the pop_receipt specified on the request must match the
pop_receipt returned from the :func:`~get_messages` or :func:`~update_message`
operation.
:param str queue_name:
The name of the queue from which to delete the message.
:param str message_id:
The message id identifying the message to delete.
:param str pop_receipt:
A valid pop receipt value returned from an earlier call
to the :func:`~get_messages` or :func:`~update_message`.
:param int timeout:
The server timeout, expressed in seconds.
'''
_validate_not_none('queue_name', queue_name)
_validate_not_none('message_id', message_id)
_validate_not_none('pop_receipt', pop_receipt)
request = HTTPRequest()
request.method = 'DELETE'
request.host_locations = self._get_host_locations()
request.path = _get_path(queue_name, True, message_id)
request.query = {
'popreceipt': _to_str(pop_receipt),
'timeout': _int_to_str(timeout)
}
self._perform_request(request)
def _encrypt_entity(entity, key_encryption_key, encryption_resolver):
'''
Encrypts the given entity using AES256 in CBC mode with 128 bit padding.
Will generate a content-encryption-key (cek) to encrypt the properties either
stored in an EntityProperty with the 'encrypt' flag set or those
specified by the encryption resolver. This cek is then wrapped using the
provided key_encryption_key (kek). Only strings may be encrypted and the
result is stored as binary on the service.
:param entity:
The entity to insert. Could be a dict or an entity object.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:param function(partition_key, row_key, property_name) encryption_resolver:
A function that takes in an entities partition key, row key, and property name and returns
a boolean that indicates whether that property should be encrypted.
:return: An entity with both the appropriate properties encrypted and the
encryption data.
:rtype: object
'''
_validate_not_none('entity', entity)
_validate_not_none('key_encryption_key', key_encryption_key)
_validate_key_encryption_key_wrap(key_encryption_key)
# AES256 uses 256 bit (32 byte) keys and always with 16 byte blocks
content_encryption_key = os.urandom(32)
entity_initialization_vector = os.urandom(16)
encrypted_properties = []
encrypted_entity = Entity()
for key, value in entity.items():
# If the property resolver says it should be encrypted
# or it is an EntityProperty with the 'encrypt' property set.
if (isinstance(value, EntityProperty) and value.encrypt) or \
(encryption_resolver is not None \
and encryption_resolver(entity['PartitionKey'], entity['RowKey'], key)):
# Only strings can be encrypted and None is not an instance of str.
if isinstance(value, EntityProperty):
if value.type == EdmType.STRING:
value = value.value
else:
raise ValueError(_ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION)
if not isinstance(value, str):
raise ValueError(_ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION)
# Value is now confirmed to hold a valid string value to be encrypted
# and should be added to the list of encrypted properties.
encrypted_properties.append(key)
propertyIV = _generate_property_iv(entity_initialization_vector,
entity['PartitionKey'], entity['RowKey'],
key, False)
# Encode the strings for encryption.
value = value.encode('utf-8')
cipher = _generate_AES_CBC_cipher(content_encryption_key, propertyIV)
# PKCS7 with 16 byte blocks ensures compatibility with AES.
padder = PKCS7(128).padder()
padded_data = padder.update(value) + padder.finalize()
# Encrypt the data.
encryptor = cipher.encryptor()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
# Set the new value of this key to be a binary EntityProperty for proper serialization.
value = EntityProperty(EdmType.BINARY, encrypted_data)
encrypted_entity[key] = value
encrypted_properties = dumps(encrypted_properties)
# Generate the metadata iv.
metadataIV = _generate_property_iv(entity_initialization_vector,
entity['PartitionKey'], entity['RowKey'],
'_ClientEncryptionMetadata2', False)
encrypted_properties = encrypted_properties.encode('utf-8')
cipher = _generate_AES_CBC_cipher(content_encryption_key, metadataIV)
padder = PKCS7(128).padder()
padded_data = padder.update(encrypted_properties) + padder.finalize()
encryptor = cipher.encryptor()
encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
encrypted_entity['_ClientEncryptionMetadata2'] = EntityProperty(EdmType.BINARY, encrypted_data)
encryption_data = _generate_encryption_data_dict(key_encryption_key, content_encryption_key,
entity_initialization_vector)
encrypted_entity['_ClientEncryptionMetadata1'] = dumps(encryption_data)
return encrypted_entity
def _put_blob(self, container_name, blob_name, blob, content_settings=None,
metadata=None, validate_content=False, lease_id=None, if_modified_since=None,
if_unmodified_since=None, if_match=None, if_none_match=None,
timeout=None):
'''
Creates a blob or updates an existing blob.
See create_blob_from_* for high level
functions that handle the creation and upload of large blobs with
automatic chunking and progress notifications.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to create or update.
:param bytes blob:
Content of blob as bytes (size < 64MB). For larger size, you
must call put_block and put_block_list to set content of blob.
:param ~azure.storage.blob.models.ContentSettings content_settings:
ContentSettings object used to set properties on the blob.
:param metadata:
Name-value pairs associated with the blob as metadata.
:param bool validate_content:
If true, calculates an MD5 hash of the blob content. The storage
service checks the hash of the content that has arrived
with the hash that was sent. This is primarily valuable for detecting
bitflips on the wire if using http instead of https as https (the default)
will already validate. Note that this MD5 hash is not stored with the
blob.
:param str lease_id:
Required if the blob has an active lease.
:param datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:param str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:param str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:param int timeout:
The timeout parameter is expressed in seconds.
:return: ETag and last modified properties for the new Block Blob
:rtype: :class:`~azure.storage.blob.models.ResourceProperties`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_encryption_required(self.require_encryption, self.key_encryption_key)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {'timeout': _int_to_str(timeout)}
request.headers = {
'x-ms-blob-type': _to_str(self.blob_type),
'x-ms-lease-id': _to_str(lease_id),
'If-Modified-Since': _datetime_to_utc_string(if_modified_since),
'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since),
'If-Match': _to_str(if_match),
'If-None-Match': _to_str(if_none_match)
}
_add_metadata_headers(metadata, request)
if content_settings is not None:
request.headers.update(content_settings._to_headers())
blob = _get_data_bytes_only('blob', blob)
if self.key_encryption_key:
encryption_data, blob = _encrypt_blob(blob, self.key_encryption_key)
request.headers['x-ms-meta-encryptiondata'] = encryption_data
request.body = blob
if validate_content:
computed_md5 = _get_content_md5(request.body)
request.headers['Content-MD5'] = _to_str(computed_md5)
return self._perform_request(request, _parse_base_properties)
def _decrypt_blob(require_encryption, key_encryption_key, key_resolver,
response, start_offset, end_offset):
'''
Decrypts the given blob contents and returns only the requested range.
:param bool require_encryption:
Whether or not the calling blob service requires objects to be decrypted.
:param object key_encryption_key:
The user-provided key-encryption-key. Must implement the following methods:
wrap_key(key)--wraps the specified key using an algorithm of the user's choice.
get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key.
get_kid()--returns a string key id for this key-encryption-key.
:param key_resolver(kid):
The user-provided key resolver. Uses the kid string to return a key-encryption-key
implementing the interface defined above.
:return: The decrypted blob content.
:rtype: bytes
'''
_validate_not_none('response', response)
content = response.body
_validate_not_none('content', content)
try:
encryption_data = _dict_to_encryption_data(loads(response.headers['x-ms-meta-encryptiondata']))
except:
if require_encryption:
raise ValueError(_ERROR_DATA_NOT_ENCRYPTED)
else:
return content
if not (encryption_data.encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_CBC_256):
raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM)
blob_type = response.headers['x-ms-blob-type']
iv = None
unpad = False
start_range, end_range = 0, len(content)
if 'content-range' in response.headers:
content_range = response.headers['content-range']
# Format: 'bytes x-y/size'
# Ignore the word 'bytes'
content_range = content_range.split(' ')
content_range = content_range[1].split('-')
start_range = int(content_range[0])
content_range = content_range[1].split('/')
end_range = int(content_range[0])
blob_size = int(content_range[1])
if start_offset >= 16:
iv = content[:16]
content = content[16:]
start_offset -= 16
else:
iv = encryption_data.content_encryption_IV
if end_range == blob_size - 1:
unpad = True
else:
unpad = True
iv = encryption_data.content_encryption_IV
if blob_type == 'PageBlob':
unpad = False
content_encryption_key = _validate_and_unwrap_cek(encryption_data, key_encryption_key, key_resolver)
cipher = _generate_AES_CBC_cipher(content_encryption_key, iv)
decryptor = cipher.decryptor()
content = decryptor.update(content) + decryptor.finalize()
if unpad:
unpadder = PKCS7(128).unpadder()
content = unpadder.update(content) + unpadder.finalize()
return content[start_offset: len(content) - end_offset]
def create_blob_from_stream(
self, container_name, blob_name, stream, count=None,
content_settings=None, metadata=None, validate_content=False,
progress_callback=None, max_connections=2, lease_id=None,
if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None, use_byte_buffer=False):
'''
Creates a new blob from a file/stream, or updates the content of
an existing blob, with automatic chunking and progress
notifications.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to create or update.
:param io.IOBase stream:
Opened file/stream to upload as the blob content.
:param int count:
Number of bytes to read from the stream. This is optional, but
should be supplied for optimal performance.
:param ~azure.storage.blob.models.ContentSettings content_settings:
ContentSettings object used to set blob properties.
:param metadata:
Name-value pairs associated with the blob as metadata.
:type metadata: dict(str, str)
:param bool validate_content:
If true, calculates an MD5 hash for each chunk of the blob. The storage
service checks the hash of the content that has arrived with the hash
that was sent. This is primarily valuable for detecting bitflips on
the wire if using http instead of https as https (the default) will
already validate. Note that this MD5 hash is not stored with the
blob.
:param progress_callback:
Callback for progress with signature function(current, total) where
current is the number of bytes transfered so far, and total is the
size of the blob, or None if the total size is unknown.
:type progress_callback: func(current, total)
:param int max_connections:
Maximum number of parallel connections to use when the blob size exceeds
64MB. Note that parallel upload requires the stream to be seekable.
:param str lease_id:
Required if the blob has an active lease.
:param datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:param str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:param str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:param int timeout:
The timeout parameter is expressed in seconds. This method may make
multiple calls to the Azure service and the timeout will apply to
each call individually.
:param bool use_byte_buffer:
If True, this will force usage of the original full block buffering upload path.
By default, this value is False and will employ a memory-efficient,
streaming upload algorithm under the following conditions:
The provided stream is seekable, 'require_encryption' is False, and
MAX_BLOCK_SIZE >= MIN_LARGE_BLOCK_UPLOAD_THRESHOLD.
One should consider the drawbacks of using this approach. In order to achieve
memory-efficiency, a IOBase stream or file-like object is segmented into logical blocks
using a SubStream wrapper. In order to read the correct data, each SubStream must acquire
a lock so that it can safely seek to the right position on the shared, underlying stream.
If max_connections > 1, the concurrency will result in a considerable amount of seeking on
the underlying stream. For the most common inputs such as a file-like stream object, seeking
is an inexpensive operation and this is not much of a concern. However, for other variants of streams
this may not be the case. The trade-off for memory-efficiency must be weighed against the cost of seeking
with your input stream.
The SubStream class will attempt to buffer up to 4 MB internally to reduce the amount of
seek and read calls to the underlying stream. This is particularly beneficial when uploading larger blocks.
:return: ETag and last modified properties for the Block Blob
:rtype: :class:`~azure.storage.blob.models.ResourceProperties`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('stream', stream)
_validate_encryption_required(self.require_encryption, self.key_encryption_key)
# Adjust count to include padding if we are expected to encrypt.
adjusted_count = count
if (self.key_encryption_key is not None) and (adjusted_count is not None):
adjusted_count += (16 - (count % 16))
if adjusted_count is not None and (adjusted_count < self.MAX_SINGLE_PUT_SIZE):
if progress_callback:
progress_callback(0, count)
data = stream.read(count)
resp = self._put_blob(
container_name=container_name,
blob_name=blob_name,
blob=data,
content_settings=content_settings,
metadata=metadata,
validate_content=validate_content,
lease_id=lease_id,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout)
if progress_callback:
progress_callback(count, count)
return resp
else:
cek, iv, encryption_data = None, None, None
use_original_upload_path = use_byte_buffer or self.require_encryption or \
self.MAX_BLOCK_SIZE < self.MIN_LARGE_BLOCK_UPLOAD_THRESHOLD or \
hasattr(stream, 'seekable') and not stream.seekable() or \
not hasattr(stream, 'seek') or not hasattr(stream, 'tell')
if use_original_upload_path:
if self.key_encryption_key:
cek, iv, encryption_data = _generate_blob_encryption_data(self.key_encryption_key)
block_ids = _upload_blob_chunks(
blob_service=self,
container_name=container_name,
blob_name=blob_name,
blob_size=count,
block_size=self.MAX_BLOCK_SIZE,
stream=stream,
max_connections=max_connections,
progress_callback=progress_callback,
validate_content=validate_content,
lease_id=lease_id,
uploader_class=_BlockBlobChunkUploader,
timeout=timeout,
content_encryption_key=cek,
initialization_vector=iv
)
else:
block_ids = _upload_blob_substream_blocks(
blob_service=self,
container_name=container_name,
blob_name=blob_name,
blob_size=count,
block_size=self.MAX_BLOCK_SIZE,
stream=stream,
max_connections=max_connections,
progress_callback=progress_callback,
validate_content=validate_content,
lease_id=lease_id,
uploader_class=_BlockBlobChunkUploader,
timeout=timeout,
)
return self._put_block_list(
container_name=container_name,
blob_name=blob_name,
block_list=block_ids,
content_settings=content_settings,
metadata=metadata,
validate_content=validate_content,
lease_id=lease_id,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout,
encryption_data=encryption_data
)
def _put_blob(self, container_name, blob_name, blob, content_settings=None,
metadata=None, validate_content=False, lease_id=None, if_modified_since=None,
if_unmodified_since=None, if_match=None, if_none_match=None,
timeout=None):
'''
Creates a blob or updates an existing blob.
See create_blob_from_* for high level
functions that handle the creation and upload of large blobs with
automatic chunking and progress notifications.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to create or update.
:param bytes blob:
Content of blob as bytes (size < 64MB). For larger size, you
must call put_block and put_block_list to set content of blob.
:param ~azure.storage.blob.models.ContentSettings content_settings:
ContentSettings object used to set properties on the blob.
:param metadata:
Name-value pairs associated with the blob as metadata.
:param bool validate_content:
If true, calculates an MD5 hash of the blob content. The storage
service checks the hash of the content that has arrived
with the hash that was sent. This is primarily valuable for detecting
bitflips on the wire if using http instead of https as https (the default)
will already validate. Note that this MD5 hash is not stored with the
blob.
:param str lease_id:
Required if the blob has an active lease.
:param datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:param str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:param str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:param int timeout:
The timeout parameter is expressed in seconds.
:return: ETag and last modified properties for the new Block Blob
:rtype: :class:`~azure.storage.blob.models.ResourceProperties`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_encryption_required(self.require_encryption, self.key_encryption_key)
request = HTTPRequest()
request.method = 'PUT'
request.host_locations = self._get_host_locations()
request.path = _get_path(container_name, blob_name)
request.query = {'timeout': _int_to_str(timeout)}
request.headers = {
'x-ms-blob-type': _to_str(self.blob_type),
'x-ms-lease-id': _to_str(lease_id),
'If-Modified-Since': _datetime_to_utc_string(if_modified_since),
'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since),
'If-Match': _to_str(if_match),
'If-None-Match': _to_str(if_none_match)
}
_add_metadata_headers(metadata, request)
if content_settings is not None:
request.headers.update(content_settings._to_headers())
blob = _get_data_bytes_only('blob', blob)
if self.key_encryption_key:
encryption_data, blob = _encrypt_blob(blob, self.key_encryption_key)
request.headers['x-ms-meta-encryptiondata'] = encryption_data
request.body = blob
if validate_content:
computed_md5 = _get_content_md5(request.body)
request.headers['Content-MD5'] = _to_str(computed_md5)
return self._perform_request(request, _parse_base_properties)
def create_blob_from_text(
self, container_name, blob_name, text, encoding='utf-8',
content_settings=None, metadata=None, validate_content=False,
progress_callback=None, max_connections=2, lease_id=None,
if_modified_since=None, if_unmodified_since=None, if_match=None,
if_none_match=None, timeout=None):
'''
Creates a new blob from str/unicode, or updates the content of an
existing blob, with automatic chunking and progress notifications.
:param str container_name:
Name of existing container.
:param str blob_name:
Name of blob to create or update.
:param str text:
Text to upload to the blob.
:param str encoding:
Python encoding to use to convert the text to bytes.
:param ~azure.storage.blob.models.ContentSettings content_settings:
ContentSettings object used to set blob properties.
:param metadata:
Name-value pairs associated with the blob as metadata.
:type metadata: dict(str, str)
:param bool validate_content:
If true, calculates an MD5 hash for each chunk of the blob. The storage
service checks the hash of the content that has arrived with the hash
that was sent. This is primarily valuable for detecting bitflips on
the wire if using http instead of https as https (the default) will
already validate. Note that this MD5 hash is not stored with the
blob.
:param progress_callback:
Callback for progress with signature function(current, total) where
current is the number of bytes transfered so far, and total is the
size of the blob, or None if the total size is unknown.
:type progress_callback: func(current, total)
:param int max_connections:
Maximum number of parallel connections to use when the blob size exceeds
64MB.
:param str lease_id:
Required if the blob has an active lease.
:param datetime if_modified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only
if the resource has been modified since the specified time.
:param datetime if_unmodified_since:
A DateTime value. Azure expects the date value passed in to be UTC.
If timezone is included, any non-UTC datetimes will be converted to UTC.
If a date is passed in without timezone info, it is assumed to be UTC.
Specify this header to perform the operation only if
the resource has not been modified since the specified date/time.
:param str if_match:
An ETag value, or the wildcard character (*). Specify this header to perform
the operation only if the resource's ETag matches the value specified.
:param str if_none_match:
An ETag value, or the wildcard character (*). Specify this header
to perform the operation only if the resource's ETag does not match
the value specified. Specify the wildcard character (*) to perform
the operation only if the resource does not exist, and fail the
operation if it does exist.
:param int timeout:
The timeout parameter is expressed in seconds. This method may make
multiple calls to the Azure service and the timeout will apply to
each call individually.
:return: ETag and last modified properties for the Block Blob
:rtype: :class:`~azure.storage.blob.models.ResourceProperties`
'''
_validate_not_none('container_name', container_name)
_validate_not_none('blob_name', blob_name)
_validate_not_none('text', text)
if not isinstance(text, bytes):
_validate_not_none('encoding', encoding)
text = text.encode(encoding)
return self.create_blob_from_bytes(
container_name=container_name,
blob_name=blob_name,
blob=text,
index=0,
count=len(text),
content_settings=content_settings,
metadata=metadata,
validate_content=validate_content,
lease_id=lease_id,
progress_callback=progress_callback,
max_connections=max_connections,
if_modified_since=if_modified_since,
if_unmodified_since=if_unmodified_since,
if_match=if_match,
if_none_match=if_none_match,
timeout=timeout)
