def test_retry_to_secondary_with_get(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_base=2).retry # Act try: service.create_container(container_name) service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status # Assert # Confirm that the get request gets retried to secondary def retry_callback(retry_context): # Only check this every other time, sometimes the secondary location fails due to delay if retry_context.count % 2 == 1: self.assertEqual(LocationMode.SECONDARY, retry_context.location_mode) service.retry_callback = retry_callback service.get_container_metadata(container_name) finally: service.response_callback = None service.retry_callback = None service.delete_container(container_name)
def test_location_lock(self): # Arrange service = self._create_storage_service(BlockBlobService, self.settings) # Act # Fail the first request and set the retry policy to retry to secondary service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_base=2).retry service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status context = _OperationContext(location_lock=True) # Assert # Confirm that the first request gets retried to secondary # The given test account must be GRS def retry_callback(retry_context): self.assertEqual(LocationMode.SECONDARY, retry_context.location_mode) service.retry_callback = retry_callback service._list_containers(prefix='lock', _context=context) # Confirm that the second list request done with the same context sticks # to the final location of the first list request (aka secondary) despite # the client normally trying primary first def request_callback(request): if self.settings.IS_EMULATED: self.assertNotEqual(-1, request.path.find('-secondary')) else: self.assertNotEqual(-1, request.host.find('-secondary')) service.request_callback = request_callback service._list_containers(prefix='lock', _context=context)
def test_secondary_location_mode(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.location_mode = LocationMode.SECONDARY service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry # Act try: service.create_container(container_name) # Override the response from secondary if it's 404 as that simply means # the container hasn't replicated. We're just testing we try secondary, # so that's fine. service.response_callback = ResponseCallback( status=404, new_status=200).override_first_status # Assert def request_callback(request): if self.settings.IS_EMULATED: self.assertNotEqual(-1, request.path.find('-secondary')) else: self.assertNotEqual(-1, request.host.find('-secondary')) service.request_callback = request_callback service.get_container_metadata(container_name) finally: # Delete will go to primary, so disable the request validation service.request_callback = None service.delete_container(container_name)
def test_retry_to_secondary_with_put(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_base=2).retry # Act try: # Fail the first create attempt service.response_callback = ResponseCallback( status=201, new_status=408).override_first_status # Assert # Confirm that the create request does *not* get retried to secondary # This should actually throw InvalidPermissions if sent to secondary, # but validate the location_mode anyways. def retry_callback(retry_context): self.assertEqual(LocationMode.PRIMARY, retry_context.location_mode) service.retry_callback = retry_callback service.create_container(container_name) finally: service.response_callback = None service.retry_callback = None service.delete_container(container_name)
def test_retry_put_block_with_non_seekable_stream(self): if TestMode.need_recording_file(self.test_mode): return # Arrange blob_name = self.get_resource_name('blob') data = self.get_random_bytes(PUT_BLOCK_SIZE) data_stream = self.NonSeekableStream(BytesIO(data)) # rig the response so that it fails for a single time self.bs.response_callback = ResponseCallback(status=201, new_status=408).override_first_status self.bs.retry = ExponentialRetry(initial_backoff=1, increment_base=2, max_attempts=3).retry # Act # Note: put_block transforms non-seekable streams into byte arrays before handing it off to the executor self.bs.put_block(self.container_name, blob_name, data_stream, 1) # Assert block_list = self.bs.get_block_list(self.container_name, blob_name, block_list_type="uncommitted") self.assertEqual(len(block_list.uncommitted_blocks), 1) self.assertEqual(block_list.uncommitted_blocks[0].size, PUT_BLOCK_SIZE) # Commit block and verify content block_list = [BlobBlock(id='1')] self.bs.put_block_list(self.container_name, blob_name, block_list) # Assert blob = self.bs.get_blob_to_bytes(self.container_name, blob_name) self.assertEqual(blob.content, data)
def test_retry_on_server_error(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) # Force the create call to 'timeout' with a 408 service.response_callback = ResponseCallback( status=201, new_status=500).override_status # Act try: created = service.create_container(container_name) finally: service.delete_container(container_name) # Assert # The initial create will return 201, but we overwrite it and retry. # The retry will then get a 409 and return false. self.assertFalse(created)
def test_retry_on_timeout(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry service.response_callback = ResponseCallback( status=201, new_status=408).override_status # Act try: created = service.create_container(container_name) finally: service.delete_container(container_name) # Assert # The initial create will return 201, but we overwrite it and retry. # The retry will then get a 409 and return false. self.assertFalse(created)
def test_retry_with_deserialization(self): # Arrange container_name = self.get_resource_name(prefix='retry') service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry try: created = service.create_container(container_name) # Act service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status containers = service.list_containers(prefix='retry') finally: service.delete_container(container_name) # Assert self.assertTrue(len(list(containers)) >= 1)
def test_retry_callback_and_retry_context(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = LinearRetry(backoff=1).retry # Force the create call to 'timeout' with a 408 service.response_callback = ResponseCallback( status=201, new_status=408).override_status def assert_exception_is_present_on_retry_context(retry_context): self.assertIsNotNone(retry_context.exception) service.retry_callback = assert_exception_is_present_on_retry_context # Act try: service.create_container(container_name) finally: service.response_callback = None service.delete_container(container_name)
def test_invalid_retry(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry # Force the create call to fail by pretending it's a teapot service.response_callback = ResponseCallback( status=201, new_status=418).override_status # Act try: service.create_container(container_name) self.fail('The callback should force failure.') except AzureHttpError as e: # Assert self.assertEqual(418, e.status_code) self.assertEqual('Created\n', e.args[0]) finally: service.delete_container(container_name)
def test_no_retry(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = no_retry # Force the create call to 'timeout' with a 408 service.response_callback = ResponseCallback( status=201, new_status=408).override_status # Act try: service.create_container(container_name) self.fail('The callback should force failure.') except AzureHttpError as e: # Assert # The call should not retry, and thus fail. self.assertEqual(408, e.status_code) self.assertEqual('Created\n', e.args[0]) finally: service.delete_container(container_name)
def test_exponential_retry(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.create_container(container_name) service.retry = ExponentialRetry(initial_backoff=1, increment_base=3, max_attempts=3).retry # Force the create call to 'timeout' with a 408 response_callback = ResponseCallback(status=200, new_status=408) service.response_callback = response_callback.override_status # Act with self.assertRaises(AzureHttpError): service.get_container_metadata(container_name) # Assert the response was called the right number of times (1 initial request + 3 retries) self.assertEqual(response_callback.count, 1 + 3) # Clean up service.response_callback = None service.delete_container(container_name)