def test_retry_to_secondary_with_get(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_power=2).retry # Act try: service.create_container(container_name) service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status # Assert # Confirm that the get request gets retried to secondary def retry_callback(retry_context): # Only check this every other time, sometimes the secondary location fails due to delay if retry_context.count % 2 == 1: self.assertEqual(LocationMode.SECONDARY, retry_context.location_mode) service.retry_callback = retry_callback service.get_container_metadata(container_name) finally: service.response_callback = None service.retry_callback = None service.delete_container(container_name)
def test_location_lock(self): # Arrange service = self._create_storage_service(BlockBlobService, self.settings) # Act # Fail the first request and set the retry policy to retry to secondary service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_power=2).retry service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status context = _OperationContext(location_lock=True) # Assert # Confirm that the first request gets retried to secondary def retry_callback(retry_context): self.assertEqual(LocationMode.SECONDARY, retry_context.location_mode) service.retry_callback = retry_callback service._list_containers(prefix='lock', _context=context) # Confirm that the second list request done with the same context sticks # to the final location of the first list request (aka secondary) despite # the client normally trying primary first def request_callback(request): self.assertNotEqual(-1, request.host.find('-secondary')) service.request_callback = request_callback service._list_containers(prefix='lock', _context=context)
def test_secondary_location_mode(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.location_mode = LocationMode.SECONDARY service.retry = ExponentialRetry(initial_backoff=1, increment_power=2).retry # Act try: service.create_container(container_name) # Override the response from secondary if it's 404 as that simply means # the container hasn't replicated. We're just testing we try secondary, # so that's fine. service.response_callback = ResponseCallback( status=404, new_status=200).override_first_status # Assert def request_callback(request): self.assertNotEqual(-1, request.host.find('-secondary')) service.request_callback = request_callback service.get_container_metadata(container_name) finally: # Delete will go to primary, so disable the request validation service.request_callback = None service.delete_container(container_name)
def test_retry_to_secondary_with_put(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_power=2).retry # Act try: # Fail the first create attempt service.response_callback = ResponseCallback( status=201, new_status=408).override_first_status # Assert # Confirm that the create request does *not* get retried to secondary # This should actually throw InvalidPermissions if sent to secondary, # but validate the location_mode anyways. def retry_callback(retry_context): self.assertEqual(LocationMode.PRIMARY, retry_context.location_mode) service.retry_callback = retry_callback service.create_container(container_name) finally: service.response_callback = None service.retry_callback = None service.delete_container(container_name)
def test_wrong_credentials(self): from azure.storage.retry import ExponentialRetry container = uuid() conn_string = \ 'DefaultEndpointsProtocol=https;AccountName={};AccountKey={}'.\ format("testaccount", "wrongsecret") store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"Incorrect padding" in str(exc.value)
def test_wrong_endpoint(self): from azure.storage.retry import ExponentialRetry container = uuid() conn_string = create_azure_conn_string(load_azure_credentials()) conn_string += \ ";BlobEndpoint=https://hopenostorethere.blob.core.windows.net;" store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"Failed to establish a new connection" in str(exc.value)
def retries(self): # By default, retries are performed with an exponential backoff. # Any custom retry logic may be used by simply defining a retry function, # but several easy pre-written options are available with modifiable settings. client = BlockBlobService(account_name='<account_name>', account_key='<account_key>') # Use an exponential retry, but modify the backoff settings # Here, we increase the initial back off, increase the number of retry attempts # and decrease the base of the exponential backoff. client.retry = ExponentialRetry(initial_backoff=30, increment_power=2, max_attempts=5).retry # Use a default linear retry policy instead client.retry = LinearRetry().retry # Turn off retries client.retry = no_retry
def test_retry_on_timeout(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_power=2).retry service.response_callback = ResponseCallback( status=201, new_status=408).override_status # Act try: created = service.create_container(container_name) finally: service.delete_container(container_name) # Assert # The initial create will return 201, but we overwrite it and retry. # The retry will then get a 409 and return false. self.assertFalse(created)
def test_retry_with_deserialization(self): # Arrange container_name = self.get_resource_name(prefix='retry') service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_power=2).retry try: created = service.create_container(container_name) # Act service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status containers = service.list_containers(prefix='retry') finally: service.delete_container(container_name) # Assert self.assertTrue(len(list(containers)) >= 1)
def test_invalid_retry(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_power=2).retry # Force the create call to fail by pretending it's a teapot service.response_callback = ResponseCallback( status=201, new_status=418).override_status # Act try: service.create_container(container_name) self.fail('The callback should force failure.') except AzureHttpError as e: # Assert self.assertEqual(418, e.status_code) self.assertEqual('Created\n', e.args[0]) finally: service.delete_container(container_name)
def test_retry_to_secondary_with_get(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(retry_to_secondary=True).retry # Act try: service.create_container(container_name) service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status # Assert # Confirm that the get request gets retried to secondary def retry_callback(retry_context): self.assertEqual(LocationMode.SECONDARY, retry_context.location_mode) service.retry_callback = retry_callback service.get_container_metadata(container_name) finally: service.delete_container(container_name)
def read_from_secondary(self): # If you are using RA-GRS accounts, you may want to enable reading from the # secondary endpoint. Note that your application will have to handle this # data potentially being out of date as the secondary may be behind the # primary. client = BlockBlobService(account_name='<account_name>', account_key='<account_key>') # The location mode is set to primary by default meaning that all requests # are sent to the primary endpoint. If you'd like to instead read from the # secondary endpoint by default, set location mode to secondary. Note that # writes will continue to go to primary as they are not allowed on secondary. client.location_mode = LocationMode.SECONDARY # You may also decide you want to retry to secondary. This is useful if # you'd like to automatically handle the primary being temporarily down. # Again, your application will have to handle data being potentially out # of date. Retry to secondary logic may be built into a custom retry policy, # but our retry policies have a flag to enable it. Here we use the same # exponential retry as by default, but allow it to retry to secondary if # the initial request to primary fails. client.location_mode = LocationMode.PRIMARY # Reset the location_mode to start with primary client.retry = ExponentialRetry(retry_to_secondary=True).retry