def _decrypt_and_deserialize_entity(entity, property_resolver, require_encryption, key_encryption_key, key_resolver): try: _validate_decryption_required(require_encryption, key_encryption_key, key_resolver) entity_iv, encrypted_properties, content_encryption_key, isJavaV1 = None, None, None, False if (key_encryption_key is not None) or (key_resolver is not None): entity_iv, encrypted_properties, content_encryption_key, isJavaV1 = \ _extract_encryption_metadata(entity, require_encryption, key_encryption_key, key_resolver) except: raise AzureException(_ERROR_DECRYPTION_FAILURE) entity = _convert_json_to_entity(entity, property_resolver, encrypted_properties) if entity_iv is not None and encrypted_properties is not None and \ content_encryption_key is not None: try: entity = _decrypt_entity(entity, encrypted_properties, content_encryption_key, entity_iv, isJavaV1) except: raise AzureException(_ERROR_DECRYPTION_FAILURE) return entity
def _parse_blob(response, name, snapshot, validate_content=False, require_encryption=False, key_encryption_key=None, key_resolver_function=None, start_offset=None, end_offset=None): if response is None: return None metadata = _parse_metadata(response) props = _parse_properties(response, BlobProperties) # For range gets, only look at 'x-ms-blob-content-md5' for overall MD5 content_settings = getattr(props, 'content_settings') if 'content-range' in response.headers: if 'x-ms-blob-content-md5' in response.headers: setattr(content_settings, 'content_md5', _to_str(response.headers['x-ms-blob-content-md5'])) else: delattr(content_settings, 'content_md5') if validate_content: computed_md5 = _get_content_md5(response.body) _validate_content_match(response.headers['content-md5'], computed_md5) if key_encryption_key is not None or key_resolver_function is not None: try: response.body = _decrypt_blob(require_encryption, key_encryption_key, key_resolver_function, response, start_offset, end_offset) except: raise AzureException(_ERROR_DECRYPTION_FAILURE) return Blob(name, snapshot, response.body, props, metadata)
def _decrypt_queue_message(message, require_encryption, key_encryption_key, resolver): ''' Returns the decrypted message contents from an EncryptedQueueMessage. If no encryption metadata is present, will return the unaltered message. :param str message: The JSON formatted QueueEncryptedMessage contents with all associated metadata. :param bool require_encryption: If set, will enforce that the retrieved messages are encrypted and decrypt them. :param object key_encryption_key: The user-provided key-encryption-key. Must implement the following methods: unwrap_key(key, algorithm)--returns the unwrapped form of the specified symmetric key using the string-specified algorithm. get_kid()--returns a string key id for this key-encryption-key. :param function resolver(kid): The user-provided key resolver. Uses the kid string to return a key-encryption-key implementing the interface defined above. :return: The plain text message from the queue message. :rtype: str ''' try: message = loads(message) encryption_data = _dict_to_encryption_data(message['EncryptionData']) decoded_data = _decode_base64_to_bytes(message['EncryptedMessageContents']) except (KeyError, ValueError): # Message was not json formatted and so was not encrypted # or the user provided a json formatted message. if require_encryption: raise ValueError(_ERROR_MESSAGE_NOT_ENCRYPTED) else: return message try: return _decrypt(decoded_data, encryption_data, key_encryption_key, resolver).decode('utf-8') except Exception: raise AzureException(_ERROR_DECRYPTION_FAILURE)
def _perform_request(self, request, encoding='utf-8'): ''' Sends the request and return response. Catches HTTPError and hands it to error handler ''' try: resp = self._filter(request) if sys.version_info >= (3,) and isinstance(resp, bytes) and \ encoding: resp = resp.decode(encoding) # Parse and wrap HTTP errors in AzureHttpError which inherits from AzureException except HTTPError as ex: _storage_error_handler(ex) # Wrap all other exceptions as AzureExceptions to ease exception handling code except Exception as ex: if sys.version_info >= (3,): # Automatic chaining in Python 3 means we keep the trace raise AzureException else: # There isn't a good solution in 2 for keeping the stack trace # in general, or that will not result in an error in 3 # However, we can keep the previous error type and message # TODO: In the future we will log the trace raise AzureException('{}: {}'.format(ex.__class__.__name__, ex.args[0])) return resp
def _perform_request(self, request, encoding='utf-8'): ''' Sends the request and return response. Catches HTTPError and hands it to error handler ''' try: response = self._filter(request) except Exception as ex: if sys.version_info >= (3, ): # Automatic chaining in Python 3 means we keep the trace raise AzureException else: # There isn't a good solution in 2 for keeping the stack trace # in general, or that will not result in an error in 3 # However, we can keep the previous error type and message # TODO: In the future we will log the trace raise AzureException('{}: {}'.format(ex.__class__.__name__, ex.args[0])) if self.response_callback: self.response_callback(response) # Parse and wrap HTTP errors in AzureHttpError which inherits from AzureException if response.status >= 300: # This exception will be caught by the general error handler # and raised as an azure http exception _storage_error_handler( HTTPError(response.status, response.message, response.headers, response.body)) return response
def _validate_echoed_client_request_id(request, response): # raise exception if the echoed client request id from the service is not identical to the one we sent if _CLIENT_REQUEST_ID_HEADER_NAME in response.headers and \ request.headers[_CLIENT_REQUEST_ID_HEADER_NAME] != response.headers[_CLIENT_REQUEST_ID_HEADER_NAME]: raise AzureException( "Echoed client request ID: {} does not match sent client request ID: {}. Service request ID: {}" .format(response.headers[_CLIENT_REQUEST_ID_HEADER_NAME], request.headers[_CLIENT_REQUEST_ID_HEADER_NAME], response.headers['x-ms-request-id']))
def _decrypt_entity(entity, encrypted_properties_list, content_encryption_key, entityIV, isJavaV1): ''' Decrypts the specified entity using AES256 in CBC mode with 128 bit padding. Unwraps the CEK using either the specified KEK or the key returned by the key_resolver. Properties specified in the encrypted_properties_list, will be decrypted and decoded to utf-8 strings. :param entity: The entity being retrieved and decrypted. Could be a dict or an entity object. :param list encrypted_properties_list: The encrypted list of all the properties that are encrypted. :param bytes[] content_encryption_key: The key used internally to encrypt the entity. Extrated from the entity metadata. :param bytes[] entityIV: The intialization vector used to seed the encryption algorithm. Extracted from the entity metadata. :return: The decrypted entity :rtype: Entity ''' _validate_not_none('entity', entity) decrypted_entity = deepcopy(entity) try: for property in entity.keys(): if property in encrypted_properties_list: value = entity[property] propertyIV = _generate_property_iv(entityIV, entity['PartitionKey'], entity['RowKey'], property, isJavaV1) cipher = _generate_AES_CBC_cipher(content_encryption_key, propertyIV) # Decrypt the property. decryptor = cipher.decryptor() decrypted_data = (decryptor.update(value.value) + decryptor.finalize()) # Unpad the data. unpadder = PKCS7(128).unpadder() decrypted_data = (unpadder.update(decrypted_data) + unpadder.finalize()) decrypted_data = decrypted_data.decode('utf-8') decrypted_entity[property] = decrypted_data decrypted_entity.pop('_ClientEncryptionMetadata1') decrypted_entity.pop('_ClientEncryptionMetadata2') return decrypted_entity except: raise AzureException(_ERROR_DECRYPTION_FAILURE)
def _perform_request(self, request, parser=None, parser_args=None, operation_context=None, expected_errors=None): ''' Sends the request and return response. Catches HTTPError and hands it to error handler ''' operation_context = operation_context or _OperationContext() retry_context = RetryContext() retry_context.is_emulated = self.is_emulated # if request body is a stream, we need to remember its current position in case retries happen if hasattr(request.body, 'read'): try: retry_context.body_position = request.body.tell() except (AttributeError, UnsupportedOperation): # if body position cannot be obtained, then retries will not work pass # Apply the appropriate host based on the location mode self._apply_host(request, operation_context, retry_context) # Apply common settings to the request _update_request(request, self._X_MS_VERSION, self._USER_AGENT_STRING) client_request_id_prefix = str.format( "Client-Request-ID={0}", request.headers['x-ms-client-request-id']) while True: try: try: # Execute the request callback if self.request_callback: self.request_callback(request) # Add date and auth after the callback so date doesn't get too old and # authentication is still correct if signed headers are added in the request # callback. This also ensures retry policies with long back offs # will work as it resets the time sensitive headers. _add_date_header(request) try: # request can be signed individually self.authentication.sign_request(request) except AttributeError: # session can also be signed self.request_session = self.authentication.signed_session( self.request_session) # Set the request context retry_context.request = request # Log the request before it goes out logger.info( "%s Outgoing request: Method=%s, Path=%s, Query=%s, Headers=%s.", client_request_id_prefix, request.method, request.path, request.query, str(request.headers).replace('\n', '')) # Perform the request response = self._httpclient.perform_request(request) # Execute the response callback if self.response_callback: self.response_callback(response) # Set the response context retry_context.response = response # Log the response when it comes back logger.info( "%s Receiving Response: " "%s, HTTP Status Code=%s, Message=%s, Headers=%s.", client_request_id_prefix, self.extract_date_and_request_id(retry_context), response.status, response.message, str(response.headers).replace('\n', '')) # Parse and wrap HTTP errors in AzureHttpError which inherits from AzureException if response.status >= 300: # This exception will be caught by the general error handler # and raised as an azure http exception _http_error_handler( HTTPError(response.status, response.message, response.headers, response.body)) # Parse the response if parser: if parser_args: args = [response] args.extend(parser_args) return parser(*args) else: return parser(response) else: return except AzureException as ex: retry_context.exception = ex raise ex except Exception as ex: retry_context.exception = ex if sys.version_info >= (3, ): # Automatic chaining in Python 3 means we keep the trace raise AzureException(ex.args[0]) else: # There isn't a good solution in 2 for keeping the stack trace # in general, or that will not result in an error in 3 # However, we can keep the previous error type and message # TODO: In the future we will log the trace msg = "" if len(ex.args) > 0: msg = ex.args[0] raise AzureException('{}: {}'.format( ex.__class__.__name__, msg)) except AzureException as ex: # only parse the strings used for logging if logging is at least enabled for CRITICAL if logger.isEnabledFor(logging.CRITICAL): exception_str_in_one_line = str(ex).replace('\n', '') status_code = retry_context.response.status if retry_context.response is not None else 'Unknown' timestamp_and_request_id = self.extract_date_and_request_id( retry_context) # if the http error was expected, we should short-circuit if isinstance( ex, AzureHttpError ) and expected_errors is not None and ex.error_code in expected_errors: logger.info( "%s Received expected http error: " "%s, HTTP status code=%s, Exception=%s.", client_request_id_prefix, timestamp_and_request_id, status_code, exception_str_in_one_line) raise ex logger.info( "%s Operation failed: checking if the operation should be retried. " "Current retry count=%s, %s, HTTP status code=%s, Exception=%s.", client_request_id_prefix, retry_context.count if hasattr( retry_context, 'count') else 0, timestamp_and_request_id, status_code, exception_str_in_one_line) # Decryption failures (invalid objects, invalid algorithms, data unencrypted in strict mode, etc) # will not be resolved with retries. if str(ex) == _ERROR_DECRYPTION_FAILURE: logger.error( "%s Encountered decryption failure: this cannot be retried. " "%s, HTTP status code=%s, Exception=%s.", client_request_id_prefix, timestamp_and_request_id, status_code, exception_str_in_one_line) raise ex # Determine whether a retry should be performed and if so, how # long to wait before performing retry. retry_interval = self.retry(retry_context) if retry_interval is not None: # Execute the callback if self.retry_callback: self.retry_callback(retry_context) logger.info( "%s Retry policy is allowing a retry: Retry count=%s, Interval=%s.", client_request_id_prefix, retry_context.count, retry_interval) # Sleep for the desired retry interval sleep(retry_interval) else: logger.error( "%s Retry policy did not allow for a retry: " "%s, HTTP status code=%s, Exception=%s.", client_request_id_prefix, timestamp_and_request_id, status_code, exception_str_in_one_line) raise ex finally: # If this is a location locked operation and the location is not set, # this is the first request of that operation. Set the location to # be used for subsequent requests in the operation. if operation_context.location_lock and not operation_context.host_location: # note: to cover the emulator scenario, the host_location is grabbed # from request.host_locations(which includes the dev account name) # instead of request.host(which at this point no longer includes the dev account name) operation_context.host_location = { retry_context.location_mode: request.host_locations[retry_context.location_mode] }
def _perform_request(self, request, parser=None, parser_args=None, operation_context=None): ''' Sends the request and return response. Catches HTTPError and hands it to error handler ''' operation_context = operation_context or _OperationContext() retry_context = RetryContext() # Apply the appropriate host based on the location mode self._apply_host(request, operation_context, retry_context) # Apply common settings to the request _update_request(request) while(True): try: try: # Execute the request callback if self.request_callback: self.request_callback(request) # Add date and auth after the callback so date doesn't get too old and # authentication is still correct if signed headers are added in the request # callback. This also ensures retry policies with long back offs # will work as it resets the time sensitive headers. _add_date_header(request) self.authentication.sign_request(request) # Set the request context retry_context.request = request # Perform the request response = self._httpclient.perform_request(request) # Execute the response callback if self.response_callback: self.response_callback(response) # Set the response context retry_context.response = response # Parse and wrap HTTP errors in AzureHttpError which inherits from AzureException if response.status >= 300: # This exception will be caught by the general error handler # and raised as an azure http exception _http_error_handler(HTTPError(response.status, response.message, response.headers, response.body)) # Parse the response if parser: if parser_args: args = [response] args.extend(parser_args) return parser(*args) else: return parser(response) else: return except AzureException as ex: raise ex except Exception as ex: if sys.version_info >= (3,): # Automatic chaining in Python 3 means we keep the trace raise AzureException(ex.args[0]) else: # There isn't a good solution in 2 for keeping the stack trace # in general, or that will not result in an error in 3 # However, we can keep the previous error type and message # TODO: In the future we will log the trace msg = "" if len(ex.args) > 0: msg = ex.args[0] raise AzureException('{}: {}'.format(ex.__class__.__name__, msg)) except AzureException as ex: # Decryption failures (invalid objects, invalid algorithms, data unencrypted in strict mode, etc) # will not be resolved with retries. if str(ex) == _ERROR_DECRYPTION_FAILURE: raise ex # Determine whether a retry should be performed and if so, how # long to wait before performing retry. retry_interval = self.retry(retry_context) if retry_interval is not None: # Execute the callback if self.retry_callback: self.retry_callback(retry_context) # Sleep for the desired retry interval sleep(retry_interval) else: raise ex finally: # If this is a location locked operation and the location is not set, # this is the first request of that operation. Set the location to # be used for subsequent requests in the operation. if operation_context.location_lock and not operation_context.host_location: operation_context.host_location = {retry_context.location_mode: request.host}
def _convert_json_to_entity(entry_element, property_resolver, encrypted_properties): ''' Convert json response to entity. The entity format is: { "Address":"Mountain View", "Age":23, "AmountDue":200.23, "*****@*****.**":"Edm.Guid", "CustomerCode":"c9da6455-213d-42c9-9a79-3e9149a57833", "*****@*****.**":"Edm.DateTime", "CustomerSince":"2008-07-10T00:00:00", "IsActive":true, "*****@*****.**":"Edm.Int64", "NumberOfOrders":"255", "PartitionKey":"mypartitionkey", "RowKey":"myrowkey" } ''' entity = Entity() properties = {} edmtypes = {} odata = {} for name, value in entry_element.items(): if name.startswith('odata.'): odata[name[6:]] = value elif name.endswith('@odata.type'): edmtypes[name[:-11]] = value else: properties[name] = value # Partition key is a known property partition_key = properties.pop('PartitionKey', None) if partition_key: entity['PartitionKey'] = partition_key # Row key is a known property row_key = properties.pop('RowKey', None) if row_key: entity['RowKey'] = row_key # Timestamp is a known property timestamp = properties.pop('Timestamp', None) if timestamp: entity['Timestamp'] = _from_entity_datetime(timestamp) for name, value in properties.items(): mtype = edmtypes.get(name) # use the property resolver if present if property_resolver: # Clients are not expected to resolve these interal fields. # This check avoids unexpected behavior from the user-defined # property resolver. if not (name == '_ClientEncryptionMetadata1' or name == '_ClientEncryptionMetadata2'): mtype = property_resolver(partition_key, row_key, name, value, mtype) # throw if the type returned is not a valid edm type if mtype and mtype not in _EDM_TYPES: raise AzureException( _ERROR_TYPE_NOT_SUPPORTED.format(mtype)) # If the property was encrypted, supercede the results of the resolver and set as binary if encrypted_properties is not None and name in encrypted_properties: mtype = EdmType.BINARY # Add type for Int32 if type(value) is int: mtype = EdmType.INT32 # no type info, property should parse automatically if not mtype: entity[name] = value else: # need an object to hold the property conv = _ENTITY_TO_PYTHON_CONVERSIONS.get(mtype) if conv is not None: try: property = conv(value) except Exception as e: # throw if the type returned by the property resolver # cannot be used in the conversion if property_resolver: raise AzureException( _ERROR_INVALID_PROPERTY_RESOLVER.format( name, value, mtype)) else: raise e else: property = EntityProperty(mtype, value) entity[name] = property # extract etag from entry etag = odata.get('etag') if timestamp: etag = 'W/"datetime\'' + url_quote(timestamp) + '\'"' entity['etag'] = etag return entity
def _validate_access_policies(identifiers): if identifiers and len(identifiers) > 5: raise AzureException(_ERROR_TOO_MANY_ACCESS_POLICIES)
def _validate_content_match(server_md5, computed_md5): if server_md5 != computed_md5: raise AzureException( _ERROR_MD5_MISMATCH.format(server_md5, computed_md5))
def throw_azure_exception(scope): """Raises azure exception.""" raise AzureException()
class AzureProviderTestCase(TestCase): """Parent Class for AzureClientFactory test cases.""" def test_name(self): """Test name property.""" obj = AzureProvider() self.assertEqual(obj.name(), 'Azure') @patch('providers.azure.provider.AzureClientFactory') def test_cost_usage_source_is_reachable_valid(self, _): """Test that cost_usage_source_is_reachable succeeds.""" credentials = { 'subscription_id': FAKE.uuid4(), 'tenant_id': FAKE.uuid4(), 'client_id': FAKE.uuid4(), 'client_secret': FAKE.word() } source_name = { 'resource_group': FAKE.word(), 'storage_account': FAKE.word() } obj = AzureProvider() self.assertTrue( obj.cost_usage_source_is_reachable(credentials, source_name)) @patch('providers.azure.provider.AzureClientFactory', side_effect=AzureException('test exception')) def test_cost_usage_source_is_reachable_exception(self, _): """Test that ValidationError is raised when AzureException is raised.""" credentials = { 'subscription_id': FAKE.uuid4(), 'tenant_id': FAKE.uuid4(), 'client_id': FAKE.uuid4(), 'client_secret': FAKE.word() } source_name = { 'resource_group': FAKE.word(), 'storage_account': FAKE.word() } with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable( credentials, source_name) def test_cost_usage_source_is_reachable_badargs(self): """Test that a ValidationError is raised when no arguments are provided.""" with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable(None, None) with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable( FAKE.word(), FAKE.word()) with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable({}, {}) def test_infra_type_implementation(self): """Test that infra type returns None.""" obj = AzureProvider() self.assertEqual( obj.infra_type_implementation(FAKE.word(), FAKE.word()), None) def test_infra_key_list_implementation(self): """Test that infra key list returns an empty list.""" obj = AzureProvider() self.assertEqual( obj.infra_key_list_implementation(FAKE.uuid4(), FAKE.word()), [])
class AzureProviderTestCase(TestCase): """Parent Class for AzureClientFactory test cases.""" def test_name(self): """Test name property.""" obj = AzureProvider() self.assertEqual(obj.name(), "Azure") @patch("providers.azure.provider.AzureClientFactory") def test_cost_usage_source_is_reachable_valid(self, _): """Test that cost_usage_source_is_reachable succeeds.""" credentials = { "subscription_id": FAKE.uuid4(), "tenant_id": FAKE.uuid4(), "client_id": FAKE.uuid4(), "client_secret": FAKE.word(), } source_name = { "resource_group": FAKE.word(), "storage_account": FAKE.word() } with patch("providers.azure.provider.AzureService") as MockHelper: MockHelper.return_value.describe_cost_management_exports.return_value = [ "report1" ] obj = AzureProvider() self.assertTrue( obj.cost_usage_source_is_reachable(credentials, source_name)) @patch("providers.azure.provider.AzureClientFactory", side_effect=AzureException("test exception")) def test_cost_usage_source_is_reachable_exception(self, _): """Test that ValidationError is raised when AzureException is raised.""" credentials = { "subscription_id": FAKE.uuid4(), "tenant_id": FAKE.uuid4(), "client_id": FAKE.uuid4(), "client_secret": FAKE.word(), } source_name = { "resource_group": FAKE.word(), "storage_account": FAKE.word() } with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable( credentials, source_name) def test_cost_usage_source_is_reachable_badargs(self): """Test that a ValidationError is raised when no arguments are provided.""" with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable(None, None) with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable( FAKE.word(), FAKE.word()) with self.assertRaises(ValidationError): AzureProvider().cost_usage_source_is_reachable({}, {}) def test_infra_type_implementation(self): """Test that infra type returns None.""" obj = AzureProvider() self.assertEqual( obj.infra_type_implementation(FAKE.word(), FAKE.word()), None) def test_infra_key_list_implementation(self): """Test that infra key list returns an empty list.""" obj = AzureProvider() self.assertEqual( obj.infra_key_list_implementation(FAKE.uuid4(), FAKE.word()), []) @patch("providers.azure.provider.AzureClientFactory") def test_cost_usage_source_reachable_without_cost_export(self, _): """Test that cost_usage_source_is_reachable raises an exception when no cost reports exist.""" credentials = { "subscription_id": FAKE.uuid4(), "tenant_id": FAKE.uuid4(), "client_id": FAKE.uuid4(), "client_secret": FAKE.word(), } source_name = { "resource_group": FAKE.word(), "storage_account": FAKE.word() } with patch("providers.azure.provider.AzureService") as MockHelper: MockHelper.return_value.describe_cost_management_exports.return_value = [] azure_provider = AzureProvider() with self.assertRaises(ValidationError): azure_provider.cost_usage_source_is_reachable( credentials, source_name)