def test_retry_to_secondary_with_put(self):
# Arrange
container_name = self.get_resource_name()
retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2)
service = self._create_storage_service(BlobServiceClient,
self.settings,
retry_policy=retry)
# Act
try:
# Fail the first create attempt
response_callback = ResponseCallback(
status=201, new_status=408).override_first_status
# Assert
# Confirm that the create request does *not* get retried to secondary
# This should actually throw InvalidPermissions if sent to secondary,
# but validate the location_mode anyways.
def retry_callback(location_mode=None, **kwargs):
self.assertEqual(LocationMode.PRIMARY, location_mode)
with self.assertRaises(ResourceExistsError):
service.create_container(container_name,
raw_response_hook=response_callback,
retry_hook=retry_callback)
finally:
service.delete_container(container_name)
async def _test_retry_put_block_with_non_seekable_stream_fail_async(self):
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
await self._setup()
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = self.NonSeekableStream(BytesIO(data))
# rig the response so that it fails for a single time
responder = ResponseCallback(status=201, new_status=408)
# Act
blob = self.bs.get_blob_client(self.container_name, blob_name)
with self.assertRaises(HttpResponseError) as error:
await blob.stage_block(
1,
data_stream,
length=PUT_BLOCK_SIZE,
raw_response_hook=responder.override_first_status)
# Assert
self.assertEqual(error.exception.response.status_code, 408)
def test_retry_callback_and_retry_context(self):
# Arrange
container_name = self.get_resource_name()
retry = LinearRetry(backoff=1)
service = self._create_storage_service(BlobServiceClient,
self.settings,
retry_policy=retry)
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=201, new_status=408).override_status
def assert_exception_is_present_on_retry_context(**kwargs):
self.assertIsNotNone(kwargs.get('response'))
self.assertEqual(kwargs['response'].status_code, 408)
# Act
try:
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
with self.assertRaises(ResourceExistsError):
service.create_container(
container_name,
raw_response_hook=callback,
retry_hook=assert_exception_is_present_on_retry_context)
finally:
service.delete_container(container_name)
def test_retry_to_secondary_with_get(self):
# Arrange
container_name = self.get_resource_name()
retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2)
service = self._create_storage_service(BlobServiceClient,
self.settings,
retry_policy=retry)
# Act
try:
container = service.create_container(container_name)
response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
# Assert
# Confirm that the get request gets retried to secondary
def retry_callback(retry_count=None, location_mode=None, **kwargs):
# Only check this every other time, sometimes the secondary location fails due to delay
if retry_count % 2 == 0:
self.assertEqual(LocationMode.SECONDARY, location_mode)
container.get_container_properties(
raw_response_hook=response_callback, retry_hook=retry_callback)
finally:
service.delete_container(container_name)
async def test_exponential_retry_async(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
container_name = self.get_resource_name('utcontainer')
retry = ExponentialRetry(initial_backoff=1,
increment_base=3,
retry_total=3)
service = self._create_storage_service(
BlobServiceClient,
storage_account,
storage_account_key,
retry_policy=retry,
transport=AiohttpTestTransport())
try:
container = await service.create_container(container_name)
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=200, new_status=408)
# Act
with self.assertRaises(HttpResponseError):
await container.get_container_properties(
raw_response_hook=callback.override_status)
# Assert the response was called the right number of times (1 initial request + 3 retries)
self.assertEqual(callback.count, 1 + 3)
finally:
# Clean up
await service.delete_container(container_name)
async def test_retry_put_block_with_non_seekable_stream_fail_async(
self, resource_group, location, storage_account,
storage_account_key):
if not self.is_live:
pytest.skip("live only")
# Arrange
bsc = BlobServiceClient(self._account_url(storage_account.name),
credential=storage_account_key,
retry_policy=self.retry,
transport=AiohttpTestTransport())
await self._setup(bsc)
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = self.NonSeekableStream(BytesIO(data))
# rig the response so that it fails for a single time
responder = ResponseCallback(status=201, new_status=408)
# Act
blob = bsc.get_blob_client(self.container_name, blob_name)
with self.assertRaises(HttpResponseError) as error:
await blob.stage_block(
1,
data_stream,
length=PUT_BLOCK_SIZE,
raw_response_hook=responder.override_first_status)
# Assert
self.assertEqual(error.exception.response.status_code, 408)
def test_retry_put_block_with_non_seekable_stream(self, resource_group, location, storage_account,
storage_account_key):
if not self.is_live:
pytest.skip("live only")
# Arrange
bsc = BlobServiceClient(self._account_url(storage_account.name), credential=storage_account_key,
retry_policy=self.retry)
self._setup(bsc)
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = self.NonSeekableStream(BytesIO(data))
# rig the response so that it fails for a single time
responder = ResponseCallback(status=201, new_status=408)
# Act
blob = bsc.get_blob_client(self.container_name, blob_name)
# Note: put_block transforms non-seekable streams into byte arrays before handing it off to the executor
blob.stage_block(1, data_stream, raw_response_hook=responder.override_first_status)
# Assert
_, uncommitted_blocks = blob.get_block_list(
block_list_type="uncommitted",
raw_response_hook=responder.override_first_status)
self.assertEqual(len(uncommitted_blocks), 1)
self.assertEqual(uncommitted_blocks[0].size, PUT_BLOCK_SIZE)
# Commit block and verify content
blob.commit_block_list(['1'], raw_response_hook=responder.override_first_status)
# Assert
content = blob.download_blob().readall()
self.assertEqual(content, data)
def test_secondary_location_mode(self):
# Arrange
container_name = self.get_resource_name()
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
service = self._create_storage_service(BlobServiceClient,
self.settings,
retry_policy=retry)
# Act
try:
container = service.create_container(container_name)
container.location_mode = LocationMode.SECONDARY
# Override the response from secondary if it's 404 as that simply means
# the container hasn't replicated. We're just testing we try secondary,
# so that's fine.
response_callback = ResponseCallback(
status=404, new_status=200).override_first_status
# Assert
def request_callback(request):
self.assertNotEqual(
-1, request.http_request.url.find('-secondary'))
request_callback = request_callback
container.get_container_properties(
raw_request_hook=request_callback,
raw_response_hook=response_callback)
finally:
# Delete will go to primary, so disable the request validation
service.delete_container(container_name)
async def _test_retry_put_block_with_seekable_stream_async(self):
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
await self._setup()
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = BytesIO(data)
# rig the response so that it fails for a single time
responder = ResponseCallback(status=201, new_status=408)
# Act
blob = self.bs.get_blob_client(self.container_name, blob_name)
await blob.stage_block(
1, data_stream, raw_response_hook=responder.override_first_status)
# Assert
_, uncommitted_blocks = await blob.get_block_list(
block_list_type="uncommitted",
raw_response_hook=responder.override_first_status)
self.assertEqual(len(uncommitted_blocks), 1)
self.assertEqual(uncommitted_blocks[0].size, PUT_BLOCK_SIZE)
# Commit block and verify content
await blob.commit_block_list(
['1'], raw_response_hook=responder.override_first_status)
# Assert
content = await (await blob.download_blob()).readall()
self.assertEqual(content, data)
def test_exponential_retry(self):
# Arrange
container_name = self.get_resource_name()
retry = ExponentialRetry(initial_backoff=1,
increment_base=3,
retry_total=3)
service = self._create_storage_service(BlobServiceClient,
self.settings,
retry_policy=retry)
try:
container = service.create_container(container_name)
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=200, new_status=408)
# Act
with self.assertRaises(HttpResponseError):
container.get_container_properties(
raw_response_hook=callback.override_status)
# Assert the response was called the right number of times (1 initial request + 3 retries)
self.assertEqual(callback.count, 1 + 3)
finally:
# Clean up
service.delete_container(container_name)
async def _test_retry_with_deserialization_async(self):
# Arrange
container_name = self.get_resource_name(prefix='retry')
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
service = self._create_storage_service(
BlobServiceClient,
self.settings,
retry_policy=retry,
transport=AiohttpTestTransport())
try:
created = await service.create_container(container_name)
# Act
callback = ResponseCallback(status=200,
new_status=408).override_first_status
containers = service.list_containers(name_starts_with='retry',
raw_response_hook=callback)
# Assert
listed = []
async for c in containers:
listed.append(c)
self.assertTrue(len(listed) >= 1)
finally:
await service.delete_container(container_name)
def test_location_lock(self):
# Arrange
retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_base=2)
service = self._create_storage_service(
BlobServiceClient, self.settings, retry_policy=retry)
# Act
# Fail the first request and set the retry policy to retry to secondary
response_callback = ResponseCallback(status=200, new_status=408).override_first_status
#context = _OperationContext(location_lock=True)
# Assert
# Confirm that the first request gets retried to secondary
# The given test account must be GRS
def retry_callback(retry_count=None, location_mode=None, **kwargs):
self.assertEqual(LocationMode.SECONDARY, location_mode)
# Confirm that the second list request done with the same context sticks
# to the final location of the first list request (aka secondary) despite
# the client normally trying primary first
requests = []
def request_callback(request):
if not requests:
requests.append(request)
else:
self.assertNotEqual(-1, request.http_request.url.find('-secondary'))
containers = service.list_containers(
results_per_page=1, retry_hook=retry_callback)
next(containers)
next(containers)
def test_retry_put_block_with_non_seekable_stream(self):
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = self.NonSeekableStream(BytesIO(data))
# rig the response so that it fails for a single time
responder = ResponseCallback(status=201, new_status=408)
# Act
blob = self.bs.get_blob_client(self.container_name, blob_name)
# Note: put_block transforms non-seekable streams into byte arrays before handing it off to the executor
blob.stage_block(1,
data_stream,
raw_response_hook=responder.override_first_status)
# Assert
_, uncommitted_blocks = blob.get_block_list(
block_list_type="uncommitted",
raw_response_hook=responder.override_first_status)
self.assertEqual(len(uncommitted_blocks), 1)
self.assertEqual(uncommitted_blocks[0].size, PUT_BLOCK_SIZE)
# Commit block and verify content
blob.commit_block_list(
['1'], raw_response_hook=responder.override_first_status)
# Assert
content = blob.download_blob().readall()
self.assertEqual(content, data)
def test_retry_on_server_error(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlobServiceClient, self.settings)
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=201, new_status=500).override_status
# Act
try:
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
with self.assertRaises(ResourceExistsError):
service.create_container(container_name, raw_response_hook=callback)
finally:
service.delete_container(container_name)
def test_retry_on_timeout(self):
# Arrange
container_name = self.get_resource_name()
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
service = self._create_storage_service(
BlobServiceClient, self.settings, retry_policy=retry)
callback = ResponseCallback(status=201, new_status=408).override_status
# Act
try:
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
with self.assertRaises(ResourceExistsError):
service.create_container(container_name, raw_response_hook=callback)
finally:
service.delete_container(container_name)
def test_invalid_retry(self):
# Arrange
container_name = self.get_resource_name()
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
service = self._create_storage_service(
BlobServiceClient, self.settings, retry_policy=retry)
# Force the create call to fail by pretending it's a teapot
callback = ResponseCallback(status=201, new_status=418).override_status
# Act
try:
with self.assertRaises(HttpResponseError) as error:
service.create_container(container_name, raw_response_hook=callback)
self.assertEqual(error.exception.response.status_code, 418)
self.assertEqual(error.exception.reason, 'Created')
finally:
service.delete_container(container_name)
def test_retry_with_deserialization(self):
# Arrange
container_name = self.get_resource_name(prefix='retry')
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
service = self._create_storage_service(
BlobServiceClient, self.settings, retry_policy=retry)
try:
created = service.create_container(container_name)
# Act
callback = ResponseCallback(status=200, new_status=408).override_first_status
containers = service.list_containers(name_starts_with='retry', raw_response_hook=callback)
# Assert
containers = list(containers)
self.assertTrue(len(containers) >= 1)
finally:
service.delete_container(container_name)
def test_no_retry(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(
BlobServiceClient, self.settings, retry_total=0)
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=201, new_status=408).override_status
# Act
try:
with self.assertRaises(HttpResponseError) as error:
service.create_container(container_name, raw_response_hook=callback)
self.assertEqual(error.exception.response.status_code, 408)
self.assertEqual(error.exception.reason, 'Created')
finally:
service.delete_container(container_name)
async def _test_linear_retry_async(self):
# Arrange
container_name = self.get_resource_name()
retry = LinearRetry(backoff=1)
service = self._create_storage_service(
BlobServiceClient,
self.settings,
retry_policy=retry,
transport=AiohttpTestTransport())
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=201, new_status=408).override_status
# Act
try:
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
with self.assertRaises(ResourceExistsError):
await service.create_container(container_name,
raw_response_hook=callback)
finally:
await service.delete_container(container_name)
async def test_retry_put_block_with_seekable_stream_async(
self, resource_group, location, storage_account,
storage_account_key):
pytest.skip("Aiohttp closes stream after request - cannot rewind.")
if not self.is_live:
pytest.skip("live only")
# Arrange
bsc = BlobServiceClient(self._account_url(storage_account.name),
credential=storage_account_key,
retry_policy=self.retry,
transport=AiohttpTestTransport())
await self._setup(bsc)
blob_name = self.get_resource_name('blob')
data = self.get_random_bytes(PUT_BLOCK_SIZE)
data_stream = BytesIO(data)
# rig the response so that it fails for a single time
responder = ResponseCallback(status=201, new_status=408)
# Act
blob = bsc.get_blob_client(self.container_name, blob_name)
await blob.stage_block(
1, data_stream, raw_response_hook=responder.override_first_status)
# Assert
_, uncommitted_blocks = await blob.get_block_list(
block_list_type="uncommitted",
raw_response_hook=responder.override_first_status)
self.assertEqual(len(uncommitted_blocks), 1)
self.assertEqual(uncommitted_blocks[0].size, PUT_BLOCK_SIZE)
# Commit block and verify content
await blob.commit_block_list(
['1'], raw_response_hook=responder.override_first_status)
# Assert
content = await (await blob.download_blob()).readall()
self.assertEqual(content, data)
def test_linear_retry(self, resource_group, location, storage_account,
storage_account_key):
# Arrange
container_name = self.get_resource_name('utcontainer')
retry = LinearRetry(backoff=1)
service = self._create_storage_service(BlobServiceClient,
storage_account,
storage_account_key,
retry_policy=retry)
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=201, new_status=408).override_status
# Act
try:
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
with self.assertRaises(ResourceExistsError):
service.create_container(container_name,
raw_response_hook=callback)
finally:
service.delete_container(container_name)
async def test_retry_on_server_error_async(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
container_name = self.get_resource_name('utcontainer')
service = self._create_storage_service(
BlobServiceClient,
storage_account,
storage_account_key,
transport=AiohttpTestTransport())
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=201, new_status=500).override_status
# Act
try:
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
with self.assertRaises(ResourceExistsError):
await service.create_container(container_name,
raw_response_hook=callback)
finally:
await service.delete_container(container_name)
async def test_no_retry_async(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
container_name = self.get_resource_name('utcontainer')
service = self._create_storage_service(
BlobServiceClient,
storage_account,
storage_account_key,
retry_total=0,
transport=AiohttpTestTransport())
# Force the create call to 'timeout' with a 408
callback = ResponseCallback(status=201, new_status=408).override_status
# Act
try:
with self.assertRaises(HttpResponseError) as error:
await service.create_container(container_name,
raw_response_hook=callback)
self.assertEqual(error.exception.response.status_code, 408)
self.assertEqual(error.exception.reason, 'Created')
finally:
await service.delete_container(container_name)
async def test_invalid_retry_async(self, resource_group, location,
storage_account, storage_account_key):
# Arrange
container_name = self.get_resource_name('utcontainer')
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
service = self._create_storage_service(
BlobServiceClient,
storage_account,
storage_account_key,
retry_policy=retry,
transport=AiohttpTestTransport())
# Force the create call to fail by pretending it's a teapot
callback = ResponseCallback(status=201, new_status=418).override_status
# Act
try:
with self.assertRaises(HttpResponseError) as error:
await service.create_container(container_name,
raw_response_hook=callback)
self.assertEqual(error.exception.response.status_code, 418)
self.assertEqual(error.exception.reason, 'Created')
finally:
await service.delete_container(container_name)
async def test_retry_on_timeout_async(self, resource_group, location,
storage_account,
storage_account_key):
# Arrange
container_name = self.get_resource_name('utcontainer')
retry = ExponentialRetry(initial_backoff=1, increment_base=2)
service = self._create_storage_service(
BlobServiceClient,
storage_account,
storage_account_key,
retry_policy=retry,
transport=AiohttpTestTransport())
callback = ResponseCallback(status=201, new_status=408).override_status
# Act
try:
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
with self.assertRaises(ResourceExistsError):
await service.create_container(container_name,
raw_response_hook=callback)
finally:
await service.delete_container(container_name)
