def test_retry_to_secondary_with_get(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2).retry
# Act
try:
service.create_container(container_name)
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
# Assert
# Confirm that the get request gets retried to secondary
def retry_callback(retry_context):
# Only check this every other time, sometimes the secondary location fails due to delay
if retry_context.count % 2 == 1:
self.assertEqual(LocationMode.SECONDARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service.get_container_metadata(container_name)
finally:
service.response_callback = None
service.retry_callback = None
service.delete_container(container_name)
def test_location_lock(self):
# Arrange
service = self._create_storage_service(BlockBlobService, self.settings)
# Act
# Fail the first request and set the retry policy to retry to secondary
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2).retry
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
context = _OperationContext(location_lock=True)
# Assert
# Confirm that the first request gets retried to secondary
# The given test account must be GRS
def retry_callback(retry_context):
self.assertEqual(LocationMode.SECONDARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service._list_containers(prefix='lock', _context=context)
# Confirm that the second list request done with the same context sticks
# to the final location of the first list request (aka secondary) despite
# the client normally trying primary first
def request_callback(request):
if self.settings.IS_EMULATED:
self.assertNotEqual(-1, request.path.find('-secondary'))
else:
self.assertNotEqual(-1, request.host.find('-secondary'))
service.request_callback = request_callback
service._list_containers(prefix='lock', _context=context)
def test_secondary_location_mode(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.location_mode = LocationMode.SECONDARY
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
# Act
try:
service.create_container(container_name)
# Override the response from secondary if it's 404 as that simply means
# the container hasn't replicated. We're just testing we try secondary,
# so that's fine.
service.response_callback = ResponseCallback(
status=404, new_status=200).override_first_status
# Assert
def request_callback(request):
if self.settings.IS_EMULATED:
self.assertNotEqual(-1, request.path.find('-secondary'))
else:
self.assertNotEqual(-1, request.host.find('-secondary'))
service.request_callback = request_callback
service.get_container_metadata(container_name)
finally:
# Delete will go to primary, so disable the request validation
service.request_callback = None
service.delete_container(container_name)
def test_retry_to_secondary_with_put(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2).retry
# Act
try:
# Fail the first create attempt
service.response_callback = ResponseCallback(
status=201, new_status=408).override_first_status
# Assert
# Confirm that the create request does *not* get retried to secondary
# This should actually throw InvalidPermissions if sent to secondary,
# but validate the location_mode anyways.
def retry_callback(retry_context):
self.assertEqual(LocationMode.PRIMARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service.create_container(container_name)
finally:
service.response_callback = None
service.retry_callback = None
service.delete_container(container_name)
def test_retry_put_block_with_non_seekable_stream(self):
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = self.NonSeekableStream(BytesIO(data))
# rig the response so that it fails for a single time
self.bs.response_callback = ResponseCallback(status=201, new_status=408).override_first_status
self.bs.retry = ExponentialRetry(initial_backoff=1, increment_base=2, max_attempts=3).retry
# Act
# Note: put_block transforms non-seekable streams into byte arrays before handing it off to the executor
self.bs.put_block(self.container_name, blob_name, data_stream, 1)
# Assert
block_list = self.bs.get_block_list(self.container_name, blob_name, block_list_type="uncommitted")
self.assertEqual(len(block_list.uncommitted_blocks), 1)
self.assertEqual(block_list.uncommitted_blocks[0].size, PUT_BLOCK_SIZE)
# Commit block and verify content
block_list = [BlobBlock(id='1')]
self.bs.put_block_list(self.container_name, blob_name, block_list)
# Assert
blob = self.bs.get_blob_to_bytes(self.container_name, blob_name)
self.assertEqual(blob.content, data)
def test_retry_on_server_error(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
# Force the create call to 'timeout' with a 408
service.response_callback = ResponseCallback(
status=201, new_status=500).override_status
# Act
try:
created = service.create_container(container_name)
finally:
service.delete_container(container_name)
# Assert
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
self.assertFalse(created)
def test_retry_on_timeout(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
service.response_callback = ResponseCallback(
status=201, new_status=408).override_status
# Act
try:
created = service.create_container(container_name)
finally:
service.delete_container(container_name)
# Assert
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
self.assertFalse(created)
def test_retry_with_deserialization(self):
# Arrange
container_name = self.get_resource_name(prefix='retry')
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
try:
created = service.create_container(container_name)
# Act
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
containers = service.list_containers(prefix='retry')
finally:
service.delete_container(container_name)
# Assert
self.assertTrue(len(list(containers)) >= 1)
def test_retry_callback_and_retry_context(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = LinearRetry(backoff=1).retry
# Force the create call to 'timeout' with a 408
service.response_callback = ResponseCallback(
status=201, new_status=408).override_status
def assert_exception_is_present_on_retry_context(retry_context):
self.assertIsNotNone(retry_context.exception)
service.retry_callback = assert_exception_is_present_on_retry_context
# Act
try:
service.create_container(container_name)
finally:
service.response_callback = None
service.delete_container(container_name)
def test_invalid_retry(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
# Force the create call to fail by pretending it's a teapot
service.response_callback = ResponseCallback(
status=201, new_status=418).override_status
# Act
try:
service.create_container(container_name)
self.fail('The callback should force failure.')
except AzureHttpError as e:
# Assert
self.assertEqual(418, e.status_code)
self.assertEqual('Created\n', e.args[0])
finally:
service.delete_container(container_name)
def test_no_retry(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = no_retry
# Force the create call to 'timeout' with a 408
service.response_callback = ResponseCallback(
status=201, new_status=408).override_status
# Act
try:
service.create_container(container_name)
self.fail('The callback should force failure.')
except AzureHttpError as e:
# Assert
# The call should not retry, and thus fail.
self.assertEqual(408, e.status_code)
self.assertEqual('Created\n', e.args[0])
finally:
service.delete_container(container_name)
def test_exponential_retry(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.create_container(container_name)
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=3,
max_attempts=3).retry
# Force the create call to 'timeout' with a 408
response_callback = ResponseCallback(status=200, new_status=408)
service.response_callback = response_callback.override_status
# Act
with self.assertRaises(AzureHttpError):
service.get_container_metadata(container_name)
# Assert the response was called the right number of times (1 initial request + 3 retries)
self.assertEqual(response_callback.count, 1 + 3)
# Clean up
service.response_callback = None
service.delete_container(container_name)