def test_retry_to_secondary_with_get(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_power=2).retry
# Act
try:
service.create_container(container_name)
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
# Assert
# Confirm that the get request gets retried to secondary
def retry_callback(retry_context):
# Only check this every other time, sometimes the secondary location fails due to delay
if retry_context.count % 2 == 1:
self.assertEqual(LocationMode.SECONDARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service.get_container_metadata(container_name)
finally:
service.response_callback = None
service.retry_callback = None
service.delete_container(container_name)
def test_location_lock(self):
# Arrange
service = self._create_storage_service(BlockBlobService, self.settings)
# Act
# Fail the first request and set the retry policy to retry to secondary
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_power=2).retry
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
context = _OperationContext(location_lock=True)
# Assert
# Confirm that the first request gets retried to secondary
def retry_callback(retry_context):
self.assertEqual(LocationMode.SECONDARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service._list_containers(prefix='lock', _context=context)
# Confirm that the second list request done with the same context sticks
# to the final location of the first list request (aka secondary) despite
# the client normally trying primary first
def request_callback(request):
self.assertNotEqual(-1, request.host.find('-secondary'))
service.request_callback = request_callback
service._list_containers(prefix='lock', _context=context)
def test_secondary_location_mode(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.location_mode = LocationMode.SECONDARY
service.retry = ExponentialRetry(initial_backoff=1,
increment_power=2).retry
# Act
try:
service.create_container(container_name)
# Override the response from secondary if it's 404 as that simply means
# the container hasn't replicated. We're just testing we try secondary,
# so that's fine.
service.response_callback = ResponseCallback(
status=404, new_status=200).override_first_status
# Assert
def request_callback(request):
self.assertNotEqual(-1, request.host.find('-secondary'))
service.request_callback = request_callback
service.get_container_metadata(container_name)
finally:
# Delete will go to primary, so disable the request validation
service.request_callback = None
service.delete_container(container_name)
def test_retry_to_secondary_with_put(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_power=2).retry
# Act
try:
# Fail the first create attempt
service.response_callback = ResponseCallback(
status=201, new_status=408).override_first_status
# Assert
# Confirm that the create request does *not* get retried to secondary
# This should actually throw InvalidPermissions if sent to secondary,
# but validate the location_mode anyways.
def retry_callback(retry_context):
self.assertEqual(LocationMode.PRIMARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service.create_container(container_name)
finally:
service.response_callback = None
service.retry_callback = None
service.delete_container(container_name)
def test_wrong_credentials(self):
from azure.storage.retry import ExponentialRetry
container = uuid()
conn_string = \
'DefaultEndpointsProtocol=https;AccountName={};AccountKey={}'.\
format("testaccount", "wrongsecret")
store = AzureBlockBlobStore(conn_string=conn_string,
container=container,
create_if_missing=False)
store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry
with pytest.raises(IOError) as exc:
store.put(u"key", b"data")
assert u"Incorrect padding" in str(exc.value)
def test_wrong_endpoint(self):
from azure.storage.retry import ExponentialRetry
container = uuid()
conn_string = create_azure_conn_string(load_azure_credentials())
conn_string += \
";BlobEndpoint=https://hopenostorethere.blob.core.windows.net;"
store = AzureBlockBlobStore(conn_string=conn_string,
container=container,
create_if_missing=False)
store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry
with pytest.raises(IOError) as exc:
store.put(u"key", b"data")
assert u"Failed to establish a new connection" in str(exc.value)
def retries(self):
# By default, retries are performed with an exponential backoff.
# Any custom retry logic may be used by simply defining a retry function,
# but several easy pre-written options are available with modifiable settings.
client = BlockBlobService(account_name='<account_name>',
account_key='<account_key>')
# Use an exponential retry, but modify the backoff settings
# Here, we increase the initial back off, increase the number of retry attempts
# and decrease the base of the exponential backoff.
client.retry = ExponentialRetry(initial_backoff=30,
increment_power=2,
max_attempts=5).retry
# Use a default linear retry policy instead
client.retry = LinearRetry().retry
# Turn off retries
client.retry = no_retry
def test_retry_on_timeout(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_power=2).retry
service.response_callback = ResponseCallback(
status=201, new_status=408).override_status
# Act
try:
created = service.create_container(container_name)
finally:
service.delete_container(container_name)
# Assert
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
self.assertFalse(created)
def test_retry_with_deserialization(self):
# Arrange
container_name = self.get_resource_name(prefix='retry')
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_power=2).retry
try:
created = service.create_container(container_name)
# Act
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
containers = service.list_containers(prefix='retry')
finally:
service.delete_container(container_name)
# Assert
self.assertTrue(len(list(containers)) >= 1)
def test_invalid_retry(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_power=2).retry
# Force the create call to fail by pretending it's a teapot
service.response_callback = ResponseCallback(
status=201, new_status=418).override_status
# Act
try:
service.create_container(container_name)
self.fail('The callback should force failure.')
except AzureHttpError as e:
# Assert
self.assertEqual(418, e.status_code)
self.assertEqual('Created\n', e.args[0])
finally:
service.delete_container(container_name)
def test_retry_to_secondary_with_get(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(retry_to_secondary=True).retry
# Act
try:
service.create_container(container_name)
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
# Assert
# Confirm that the get request gets retried to secondary
def retry_callback(retry_context):
self.assertEqual(LocationMode.SECONDARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service.get_container_metadata(container_name)
finally:
service.delete_container(container_name)
def read_from_secondary(self):
# If you are using RA-GRS accounts, you may want to enable reading from the
# secondary endpoint. Note that your application will have to handle this
# data potentially being out of date as the secondary may be behind the
# primary.
client = BlockBlobService(account_name='<account_name>',
account_key='<account_key>')
# The location mode is set to primary by default meaning that all requests
# are sent to the primary endpoint. If you'd like to instead read from the
# secondary endpoint by default, set location mode to secondary. Note that
# writes will continue to go to primary as they are not allowed on secondary.
client.location_mode = LocationMode.SECONDARY
# You may also decide you want to retry to secondary. This is useful if
# you'd like to automatically handle the primary being temporarily down.
# Again, your application will have to handle data being potentially out
# of date. Retry to secondary logic may be built into a custom retry policy,
# but our retry policies have a flag to enable it. Here we use the same
# exponential retry as by default, but allow it to retry to secondary if
# the initial request to primary fails.
client.location_mode = LocationMode.PRIMARY # Reset the location_mode to start with primary
client.retry = ExponentialRetry(retry_to_secondary=True).retry
