def _get_entity(partition_key, row_key, select, accept): ''' Constructs a get entity request. ''' _validate_not_none('partition_key', partition_key) _validate_not_none('row_key', row_key) _validate_not_none('accept', accept) request = HTTPRequest() request.method = 'GET' request.headers = {'Accept': _to_str(accept)} request.query = {'$select': _to_str(select)} return request
def _parse_blob(response, name, snapshot, validate_content=False, require_encryption=False, key_encryption_key=None, key_resolver_function=None, start_offset=None, end_offset=None): if response is None: return None metadata = _parse_metadata(response) props = _parse_properties(response, BlobProperties) # For range gets, only look at 'x-ms-blob-content-md5' for overall MD5 content_settings = getattr(props, 'content_settings') if 'content-range' in response.headers: if 'x-ms-blob-content-md5' in response.headers: setattr(content_settings, 'content_md5', _to_str(response.headers['x-ms-blob-content-md5'])) else: delattr(content_settings, 'content_md5') if validate_content: computed_md5 = _get_content_md5(response.body) _validate_content_match(response.headers['content-md5'], computed_md5) if key_encryption_key is not None or key_resolver_function is not None: try: response.body = _decrypt_blob(require_encryption, key_encryption_key, key_resolver_function, response, start_offset, end_offset) except: raise AzureException(_ERROR_DECRYPTION_FAILURE) return Blob(name, snapshot, response.body, props, metadata)
def _update_entity(entity, if_match, encryption_required=False, key_encryption_key=None, encryption_resolver=None): ''' Constructs an update entity request. :param entity: The entity to insert. Could be a dict or an entity object. :param object key_encryption_key: The user-provided key-encryption-key. Must implement the following methods: wrap_key(key)--wraps the specified key using an algorithm of the user's choice. get_key_wrap_algorithm()--returns the algorithm used to wrap the specified symmetric key. get_kid()--returns a string key id for this key-encryption-key. :param function(partition_key, row_key, property_name) encryption_resolver: A function that takes in an entities partition key, row key, and property name and returns a boolean that indicates whether that property should be encrypted. ''' _validate_not_none('if_match', if_match) _validate_entity(entity, key_encryption_key is not None) _validate_encryption_required(encryption_required, key_encryption_key) request = HTTPRequest() request.method = 'PUT' request.headers = { _DEFAULT_CONTENT_TYPE_HEADER[0]: _DEFAULT_CONTENT_TYPE_HEADER[1], _DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1], 'If-Match': _to_str(if_match), } if key_encryption_key: entity = _encrypt_entity(entity, key_encryption_key, encryption_resolver) request.body = _get_request_body(_convert_entity_to_json(entity)) return request
async def get_messages(self, queue_name, num_messages=None, visibility_timeout=None, timeout=None): ''' Retrieves one or more messages from the front of the queue. When a message is retrieved from the queue, the response includes the message content and a pop_receipt value, which is required to delete the message. The message is not automatically deleted from the queue, but after it has been retrieved, it is not visible to other clients for the time interval specified by the visibility_timeout parameter. If the key-encryption-key or resolver field is set on the local service object, the messages will be decrypted before being returned. :param str queue_name: The name of the queue to get messages from. :param int num_messages: A nonzero integer value that specifies the number of messages to retrieve from the queue, up to a maximum of 32. If fewer are visible, the visible messages are returned. By default, a single message is retrieved from the queue with this operation. :param int visibility_timeout: Specifies the new visibility timeout value, in seconds, relative to server time. The new value must be larger than or equal to 1 second, and cannot be larger than 7 days. The visibility timeout of a message can be set to a value later than the expiry time. :param int timeout: The server timeout, expressed in seconds. :return: A :class:`~azure.azure_async.queue.models.QueueMessage` object representing the information passed. :rtype: list(:class:`~azure.azure_async.queue.models.QueueMessage`) ''' _validate_decryption_required(self.require_encryption, self.key_encryption_key, self.key_resolver_function) _validate_not_none('queue_name', queue_name) request = HTTPRequest() request.method = 'GET' request.host_locations = self._get_host_locations() request.path = _get_path(queue_name, True) request.query = { 'numofmessages': _to_str(num_messages), 'visibilitytimeout': _to_str(visibility_timeout), 'timeout': _int_to_str(timeout) } return await self._perform_request(request, _convert_xml_to_queue_messages, [self.decode_function, self.require_encryption, self.key_encryption_key, self.key_resolver_function])
async def _list_queues(self, prefix=None, marker=None, max_results=None, include=None, timeout=None, _context=None): ''' Returns a list of queues under the specified account. Makes a single list request to the service. Used internally by the list_queues method. :param str prefix: Filters the results to return only queues with names that begin with the specified prefix. :param str marker: A token which identifies the portion of the query to be returned with the next query operation. The operation returns a next_marker element within the response body if the list returned was not complete. This value may then be used as a query parameter in a subsequent call to request the next portion of the list of queues. The marker value is opaque to the client. :param int max_results: The maximum number of queues to return. A single list request may return up to 1000 queues and potentially a continuation token which should be followed to get additional resutls. :param str include: Include this parameter to specify that the container's metadata be returned as part of the response body. :param int timeout: The server timeout, expressed in seconds. ''' request = HTTPRequest() request.method = 'GET' request.host_locations = self._get_host_locations(secondary=True) request.path = _get_path() request.query = { 'comp': 'list', 'prefix': _to_str(prefix), 'marker': _to_str(marker), 'maxresults': _int_to_str(max_results), 'include': _to_str(include), 'timeout': _int_to_str(timeout) } return await self._perform_request(request, _convert_xml_to_queues, operation_context=_context)
def _to_headers(self): return { 'x-ms-blob-cache-control': _to_str(self.cache_control), 'x-ms-blob-content-type': _to_str(self.content_type), 'x-ms-blob-content-disposition': _to_str(self.content_disposition), 'x-ms-blob-content-md5': _to_str(self.content_md5), 'x-ms-blob-content-encoding': _to_str(self.content_encoding), 'x-ms-blob-content-language': _to_str(self.content_language), }
def _delete_entity(partition_key, row_key, if_match): ''' Constructs a delete entity request. ''' _validate_not_none('if_match', if_match) _validate_not_none('partition_key', partition_key) _validate_not_none('row_key', row_key) request = HTTPRequest() request.method = 'DELETE' request.headers = { _DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1], 'If-Match': _to_str(if_match) } return request
async def peek_messages(self, queue_name, num_messages=None, timeout=None): ''' Retrieves one or more messages from the front of the queue, but does not alter the visibility of the message. Only messages that are visible may be retrieved. When a message is retrieved for the first time with a call to get_messages, its dequeue_count property is set to 1. If it is not deleted and is subsequently retrieved again, the dequeue_count property is incremented. The client may use this value to determine how many times a message has been retrieved. Note that a call to peek_messages does not increment the value of DequeueCount, but returns this value for the client to read. If the key-encryption-key or resolver field is set on the local service object, the messages will be decrypted before being returned. :param str queue_name: The name of the queue to peek messages from. :param int num_messages: A nonzero integer value that specifies the number of messages to peek from the queue, up to a maximum of 32. By default, a single message is peeked from the queue with this operation. :param int timeout: The server timeout, expressed in seconds. :return: A list of :class:`~azure.azure_async.queue.models.QueueMessage` objects. Note that time_next_visible and pop_receipt will not be populated as peek does not pop the message and can only retrieve already visible messages. :rtype: list(:class:`~azure.azure_async.queue.models.QueueMessage`) ''' _validate_decryption_required(self.require_encryption, self.key_encryption_key, self.key_resolver_function) _validate_not_none('queue_name', queue_name) request = HTTPRequest() request.method = 'GET' request.host_locations = self._get_host_locations(secondary=True) request.path = _get_path(queue_name, True) request.query = { 'peekonly': 'true', 'numofmessages': _to_str(num_messages), 'timeout': _int_to_str(timeout) } return await self._perform_request(request, _convert_xml_to_queue_messages, [self.decode_function, self.require_encryption, self.key_encryption_key, self.key_resolver_function])
def _merge_entity(entity, if_match, require_encryption=False, key_encryption_key=None): ''' Constructs a merge entity request. ''' _validate_not_none('if_match', if_match) _validate_entity(entity) _validate_encryption_unsupported(require_encryption, key_encryption_key) request = HTTPRequest() request.method = 'MERGE' request.headers = { _DEFAULT_CONTENT_TYPE_HEADER[0]: _DEFAULT_CONTENT_TYPE_HEADER[1], _DEFAULT_ACCEPT_HEADER[0]: _DEFAULT_ACCEPT_HEADER[1], 'If-Match': _to_str(if_match) } request.body = _get_request_body(_convert_entity_to_json(entity)) return request
async def delete_message(self, queue_name, message_id, pop_receipt, timeout=None): ''' Deletes the specified message. Normally after a client retrieves a message with the get_messages operation, the client is expected to process and delete the message. To delete the message, you must have two items of data: id and pop_receipt. The id is returned from the previous get_messages operation. The pop_receipt is returned from the most recent :func:`~get_messages` or :func:`~update_message` operation. In order for the delete_message operation to succeed, the pop_receipt specified on the request must match the pop_receipt returned from the :func:`~get_messages` or :func:`~update_message` operation. :param str queue_name: The name of the queue from which to delete the message. :param str message_id: The message id identifying the message to delete. :param str pop_receipt: A valid pop receipt value returned from an earlier call to the :func:`~get_messages` or :func:`~update_message`. :param int timeout: The server timeout, expressed in seconds. ''' _validate_not_none('queue_name', queue_name) _validate_not_none('message_id', message_id) _validate_not_none('pop_receipt', pop_receipt) request = HTTPRequest() request.method = 'DELETE' request.host_locations = self._get_host_locations() request.path = _get_path(queue_name, True, message_id) self._set_basic_headers(request) request.query = { 'popreceipt': _to_str(pop_receipt), 'timeout': _int_to_str(timeout) } await self._perform_request(request)
def _convert_xml_to_blob_list(response): ''' <?xml version="1.0" encoding="utf-8"?> <EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer"> <Prefix>string-value</Prefix> <Marker>string-value</Marker> <MaxResults>int-value</MaxResults> <Delimiter>string-value</Delimiter> <Blobs> <Blob> <Name>blob-name</name> <Snapshot>date-time-value</Snapshot> <Properties> <Last-Modified>date-time-value</Last-Modified> <Etag>etag</Etag> <Content-Length>size-in-bytes</Content-Length> <Content-Type>blob-content-type</Content-Type> <Content-Encoding /> <Content-Language /> <Content-MD5 /> <Cache-Control /> <x-ms-blob-sequence-number>sequence-number</x-ms-blob-sequence-number> <BlobType>BlockBlob|PageBlob|AppendBlob</BlobType> <LeaseStatus>locked|unlocked</LeaseStatus> <LeaseState>available | leased | expired | breaking | broken</LeaseState> <LeaseDuration>infinite | fixed</LeaseDuration> <CopyId>id</CopyId> <CopyStatus>pending | success | aborted | failed </CopyStatus> <CopySource>source url</CopySource> <CopyProgress>bytes copied/bytes total</CopyProgress> <CopyCompletionTime>datetime</CopyCompletionTime> <CopyStatusDescription>error string</CopyStatusDescription> <AccessTier>P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60 | Archive | Cool | Hot</AccessTier> <AccessTierChangeTime>date-time-value</AccessTierChangeTime> <AccessTierInferred>true</AccessTierInferred> </Properties> <Metadata> <Name>value</Name> </Metadata> </Blob> <BlobPrefix> <Name>blob-prefix</Name> </BlobPrefix> </Blobs> <NextMarker /> </EnumerationResults> ''' if response is None or response.body is None: return None blob_list = _list() list_element = ETree.fromstring(response.body) setattr(blob_list, 'next_marker', list_element.findtext('NextMarker')) blobs_element = list_element.find('Blobs') blob_prefix_elements = blobs_element.findall('BlobPrefix') if blob_prefix_elements is not None: for blob_prefix_element in blob_prefix_elements: prefix = BlobPrefix() prefix.name = blob_prefix_element.findtext('Name') blob_list.append(prefix) for blob_element in blobs_element.findall('Blob'): blob = Blob() blob.name = blob_element.findtext('Name') blob.snapshot = blob_element.findtext('Snapshot') # Properties properties_element = blob_element.find('Properties') if properties_element is not None: for property_element in properties_element: info = LIST_BLOBS_ATTRIBUTE_MAP.get(property_element.tag) if info is None: setattr(blob.properties, property_element.tag, _to_str(property_element.text)) elif info[0] is None: setattr(blob.properties, info[1], info[2](property_element.text)) else: attr = getattr(blob.properties, info[0]) setattr(attr, info[1], info[2](property_element.text)) # Metadata metadata_root_element = blob_element.find('Metadata') if metadata_root_element is not None: blob.metadata = dict() for metadata_element in metadata_root_element: blob.metadata[metadata_element.tag] = metadata_element.text # Add blob to list blob_list.append(blob) return blob_list
def _get_entity_path(table_name, partition_key, row_key): return '/{0}(PartitionKey=\'{1}\',RowKey=\'{2}\')'.format( _to_str(table_name), _to_str(partition_key), _to_str(row_key))
async def append_block(self, container_name, blob_name, block, validate_content=False, maxsize_condition=None, appendpos_condition=None, lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None): ''' Commits a new block of data to the end of an existing append blob. :param str container_name: Name of existing container. :param str blob_name: Name of existing blob. :param bytes block: Content of the block in bytes. :param bool validate_content: If true, calculates an MD5 hash of the block content. The azure_async service checks the hash of the content that has arrived with the hash that was sent. This is primarily valuable for detecting bitflips on the wire if using http instead of https as https (the default) will already validate. Note that this MD5 hash is not stored with the blob. :param int maxsize_condition: Optional conditional header. The max length in bytes permitted for the append blob. If the Append Block operation would cause the blob to exceed that limit or if the blob size is already greater than the value specified in this header, the request will fail with MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). :param int appendpos_condition: Optional conditional header, used only for the Append Block operation. A number indicating the byte offset to compare. Append Block will succeed only if the append position is equal to this number. If it is not, the request will fail with the AppendPositionConditionNotMet error (HTTP status code 412 - Precondition Failed). :param str lease_id: Required if the blob has an active lease. :param datetime if_modified_since: A DateTime value. Azure expects the date value passed in to be UTC. If timezone is included, any non-UTC datetimes will be converted to UTC. If a date is passed in without timezone info, it is assumed to be UTC. Specify this header to perform the operation only if the resource has been modified since the specified time. :param datetime if_unmodified_since: A DateTime value. Azure expects the date value passed in to be UTC. If timezone is included, any non-UTC datetimes will be converted to UTC. If a date is passed in without timezone info, it is assumed to be UTC. Specify this header to perform the operation only if the resource has not been modified since the specified date/time. :param str if_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag matches the value specified. :param str if_none_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag does not match the value specified. Specify the wildcard character (*) to perform the operation only if the resource does not exist, and fail the operation if it does exist. :param int timeout: The timeout parameter is expressed in seconds. :return: ETag, last modified, append offset, and committed block count properties for the updated Append Blob :rtype: :class:`~azure.azure_async.blob.models.AppendBlockProperties` ''' _validate_not_none('container_name', container_name) _validate_not_none('blob_name', blob_name) _validate_not_none('block', block) _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) request = HTTPRequest() request.method = 'PUT' request.host_locations = self._get_host_locations() request.path = _get_path(container_name, blob_name) request.query = { 'comp': 'appendblock', 'timeout': _int_to_str(timeout), } request.headers = { 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Host': f'{self.account_name}.blob.core.windows.net', 'Content-Type': 'application/octet-stream', 'x-ms-blob-condition-maxsize': _to_str(maxsize_condition), 'x-ms-blob-condition-appendpos': _to_str(appendpos_condition), 'x-ms-lease-id': _to_str(lease_id), 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), 'If-Match': _to_str(if_match), 'If-None-Match': _to_str(if_none_match) } request.body = _get_data_bytes_only('block', block) if validate_content: computed_md5 = _get_content_md5(request.body) request.headers['Content-MD5'] = _to_str(computed_md5) return await self._perform_request(request, _parse_append_block)
async def create_blob(self, container_name, blob_name, content_settings=None, metadata=None, lease_id=None, if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None): ''' Creates a blob or overrides an existing blob. Use if_match=* to prevent overriding an existing blob. See create_blob_from_* for high level functions that handle the creation and upload of large blobs with automatic chunking and progress notifications. :param str container_name: Name of existing container. :param str blob_name: Name of blob to create or update. :param ~azure.azure_async.blob.models.ContentSettings content_settings: ContentSettings object used to set blob properties. :param metadata: Name-value pairs associated with the blob as metadata. :type metadata: dict(str, str) :param str lease_id: Required if the blob has an active lease. :param datetime if_modified_since: A DateTime value. Azure expects the date value passed in to be UTC. If timezone is included, any non-UTC datetimes will be converted to UTC. If a date is passed in without timezone info, it is assumed to be UTC. Specify this header to perform the operation only if the resource has been modified since the specified time. :param datetime if_unmodified_since: A DateTime value. Azure expects the date value passed in to be UTC. If timezone is included, any non-UTC datetimes will be converted to UTC. If a date is passed in without timezone info, it is assumed to be UTC. Specify this header to perform the operation only if the resource has not been modified since the specified date/time. :param str if_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag matches the value specified. :param str if_none_match: An ETag value, or the wildcard character (*). Specify this header to perform the operation only if the resource's ETag does not match the value specified. Specify the wildcard character (*) to perform the operation only if the resource does not exist, and fail the operation if it does exist. :param int timeout: The timeout parameter is expressed in seconds. :return: ETag and last modified properties for the updated Append Blob :rtype: :class:`~azure.azure_async.blob.models.ResourceProperties` ''' _validate_not_none('container_name', container_name) _validate_not_none('blob_name', blob_name) _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) request = HTTPRequest() request.method = 'PUT' request.host_locations = self._get_host_locations() request.path = _get_path(container_name, blob_name) request.query = {'timeout': _int_to_str(timeout)} request.headers = { 'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'Host': f'{self.account_name}.blob.core.windows.net', 'Content-Type': 'application/octet-stream', 'x-ms-blob-type': _to_str(self.blob_type), 'x-ms-lease-id': _to_str(lease_id), 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), 'If-Match': _to_str(if_match), 'If-None-Match': _to_str(if_none_match) } _add_metadata_headers(metadata, request) if content_settings is not None: request.headers.update(content_settings._to_headers()) return await self._perform_request(request, _parse_base_properties)
async def update_message(self, queue_name, message_id, pop_receipt, visibility_timeout, content=None, timeout=None): ''' Updates the visibility timeout of a message. You can also use this operation to update the contents of a message. This operation can be used to continually extend the invisibility of a queue message. This functionality can be useful if you want a worker role to "lease" a queue message. For example, if a worker role calls get_messages and recognizes that it needs more time to process a message, it can continually extend the message's invisibility until it is processed. If the worker role were to fail during processing, eventually the message would become visible again and another worker role could process it. If the key-encryption-key field is set on the local service object, this method will encrypt the content before uploading. :param str queue_name: The name of the queue containing the message to update. :param str message_id: The message id identifying the message to update. :param str pop_receipt: A valid pop receipt value returned from an earlier call to the :func:`~get_messages` or :func:`~update_message` operation. :param int visibility_timeout: Specifies the new visibility timeout value, in seconds, relative to server time. The new value must be larger than or equal to 0, and cannot be larger than 7 days. The visibility timeout of a message cannot be set to a value later than the expiry time. A message can be updated until it has been deleted or has expired. :param obj content: Message content. Allowed type is determined by the encode_function set on the service. Default is str. :param int timeout: The server timeout, expressed in seconds. :return: A list of :class:`~azure.azure_async.queue.models.QueueMessage` objects. For convenience, this object is also populated with the content, although it is not returned by the service. :rtype: list(:class:`~azure.azure_async.queue.models.QueueMessage`) ''' _validate_encryption_required(self.require_encryption, self.key_encryption_key) _validate_not_none('queue_name', queue_name) _validate_not_none('message_id', message_id) _validate_not_none('pop_receipt', pop_receipt) _validate_not_none('visibility_timeout', visibility_timeout) request = HTTPRequest() request.method = 'PUT' request.host_locations = self._get_host_locations() self._set_basic_headers(request) request.path = _get_path(queue_name, True, message_id) request.query = { 'popreceipt': _to_str(pop_receipt), 'visibilitytimeout': _int_to_str(visibility_timeout), 'timeout': _int_to_str(timeout) } if content is not None: request.body = _get_request_body(_convert_queue_message_xml(content, self.encode_function, self.key_encryption_key)) return await self._perform_request(request, _parse_queue_message_from_headers)
async def put_message(self, queue_name, content, visibility_timeout=None, time_to_live=None, timeout=None): ''' Adds a new message to the back of the message queue. The visibility timeout specifies the time that the message will be invisible. After the timeout expires, the message will become visible. If a visibility timeout is not specified, the default value of 0 is used. The message time-to-live specifies how long a message will remain in the queue. The message will be deleted from the queue when the time-to-live period expires. If the key-encryption-key field is set on the local service object, this method will encrypt the content before uploading. :param str queue_name: The name of the queue to put the message into. :param obj content: Message content. Allowed type is determined by the encode_function set on the service. Default is str. The encoded message can be up to 64KB in size. :param int visibility_timeout: If not specified, the default value is 0. Specifies the new visibility timeout value, in seconds, relative to server time. The value must be larger than or equal to 0, and cannot be larger than 7 days. The visibility timeout of a message cannot be set to a value later than the expiry time. visibility_timeout should be set to a value smaller than the time-to-live value. :param int time_to_live: Specifies the time-to-live interval for the message, in seconds. The maximum time-to-live allowed is 7 days. If this parameter is omitted, the default time-to-live is 7 days. :param int timeout: The server timeout, expressed in seconds. :return: A :class:`~azure.azure_async.queue.models.QueueMessage` object. This object is also populated with the content although it is not returned from the service. :rtype: :class:`~azure.azure_async.queue.models.QueueMessage` ''' _validate_encryption_required(self.require_encryption, self.key_encryption_key) _validate_not_none('queue_name', queue_name) _validate_not_none('content', content) request = HTTPRequest() request.method = 'POST' request.host_locations = self._get_host_locations() request.path = _get_path(queue_name, True) self._set_basic_headers(request) request.query = { 'visibilitytimeout': _to_str(visibility_timeout), 'messagettl': _to_str(time_to_live), 'timeout': _int_to_str(timeout) } request.body = _get_request_body(_convert_queue_message_xml(content, self.encode_function, self.key_encryption_key)) message_list = await self._perform_request(request, _convert_xml_to_queue_messages, [self.decode_function, False, None, None, content]) return message_list[0]