def test_exponential_retry_interval(self): # Arrange initial_backoff = 15 increment_power = 3 retry_policy = ExponentialRetry(initial_backoff, increment_power) context_stub = RetryContext() # Act context_stub.count = 0 backoff = retry_policy._backoff(context_stub) # Assert self.assertEqual(backoff, 15) # Act context_stub.count = 1 backoff = retry_policy._backoff(context_stub) # Assert self.assertEqual(backoff, 18) # Act context_stub.count = 2 backoff = retry_policy._backoff(context_stub) # Assert self.assertEqual(backoff, 24) # Act context_stub.count = 3 backoff = retry_policy._backoff(context_stub) # Assert self.assertEqual(backoff, 42)
def create_client(stage_info, use_accelerate_endpoint=False): """ Creates a client object with a stage credential :param stage_credentials: a stage credential :param use_accelerate_endpoint: is accelerate endpoint? :return: client """ stage_credentials = stage_info[u'creds'] sas_token = stage_credentials[u'AZURE_SAS_TOKEN'] if sas_token and sas_token.startswith(u'?'): sas_token = sas_token[1:] end_point = stage_info['endPoint'] if end_point.startswith('blob.'): end_point = end_point[len('blob.'):] client = BlockBlobService(account_name=stage_info[u'storageAccount'], sas_token=sas_token, endpoint_suffix=end_point) client._httpclient = RawBodyReadingClient(session=requests.session(), protocol="https", timeout=2000) client.retry = ExponentialRetry(initial_backoff=1, increment_base=2, max_attempts=60, random_jitter_range=2).retry return client
def test_secondary_location_mode(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.location_mode = LocationMode.SECONDARY service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry # Act try: service.create_container(container_name) # Override the response from secondary if it's 404 as that simply means # the container hasn't replicated. We're just testing we try secondary, # so that's fine. service.response_callback = ResponseCallback( status=404, new_status=200).override_first_status # Assert def request_callback(request): if self.settings.IS_EMULATED: self.assertNotEqual(-1, request.path.find('-secondary')) else: self.assertNotEqual(-1, request.host.find('-secondary')) service.request_callback = request_callback service.get_container_metadata(container_name) finally: # Delete will go to primary, so disable the request validation service.request_callback = None service.delete_container(container_name)
def test_retry_to_secondary_with_put(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_base=2).retry # Act try: # Fail the first create attempt service.response_callback = ResponseCallback( status=201, new_status=408).override_first_status # Assert # Confirm that the create request does *not* get retried to secondary # This should actually throw InvalidPermissions if sent to secondary, # but validate the location_mode anyways. def retry_callback(retry_context): self.assertEqual(LocationMode.PRIMARY, retry_context.location_mode) service.retry_callback = retry_callback service.create_container(container_name) finally: service.response_callback = None service.retry_callback = None service.delete_container(container_name)
def test_retry_to_secondary_with_get(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_base=2).retry # Act try: service.create_container(container_name) service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status # Assert # Confirm that the get request gets retried to secondary def retry_callback(retry_context): # Only check this every other time, sometimes the secondary location fails due to delay if retry_context.count % 2 == 1: self.assertEqual(LocationMode.SECONDARY, retry_context.location_mode) service.retry_callback = retry_callback service.get_container_metadata(container_name) finally: service.response_callback = None service.retry_callback = None service.delete_container(container_name)
def test_location_lock(self): # Arrange service = self._create_storage_service(BlockBlobService, self.settings) # Act # Fail the first request and set the retry policy to retry to secondary service.retry = ExponentialRetry(retry_to_secondary=True, initial_backoff=1, increment_base=2).retry service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status context = _OperationContext(location_lock=True) # Assert # Confirm that the first request gets retried to secondary # The given test account must be GRS def retry_callback(retry_context): self.assertEqual(LocationMode.SECONDARY, retry_context.location_mode) service.retry_callback = retry_callback service._list_containers(prefix='lock', _context=context) # Confirm that the second list request done with the same context sticks # to the final location of the first list request (aka secondary) despite # the client normally trying primary first def request_callback(request): if self.settings.IS_EMULATED: self.assertNotEqual(-1, request.path.find('-secondary')) else: self.assertNotEqual(-1, request.host.find('-secondary')) service.request_callback = request_callback service._list_containers(prefix='lock', _context=context)
def __init__(self): with open('./vardhamanbot/bot/config/app.json') as app_config_file: app_config = json.load(app_config_file) self.tableservice = TableService(app_config["ats_name"], app_config["ats_key"]) self.tableservice.retry = ExponentialRetry(initial_backoff=30, increment_base=2, max_attempts=20).retry
def test_exponential_retry_interval(self): # Arrange retry_policy = ExponentialRetry(initial_backoff=1, increment_base=3, random_jitter_range=3) context_stub = RetryContext() for i in range(10): # Act context_stub.count = 0 backoff = retry_policy._backoff(context_stub) # Assert backoff interval is within +/- 3 of 1 self.assertTrue(0 <= backoff <= 4) # Act context_stub.count = 1 backoff = retry_policy._backoff(context_stub) # Assert backoff interval is within +/- 3 of 4(1+3^1) self.assertTrue(1 <= backoff <= 7) # Act context_stub.count = 2 backoff = retry_policy._backoff(context_stub) # Assert backoff interval is within +/- 3 of 10(1+3^2) self.assertTrue(7 <= backoff <= 13) # Act context_stub.count = 3 backoff = retry_policy._backoff(context_stub) # Assert backoff interval is within +/- 3 of 28(1+3^3) self.assertTrue(25 <= backoff <= 31)
def test_wrong_credentials(self): from azure.storage.common.retry import ExponentialRetry container = uuid() conn_string = \ 'DefaultEndpointsProtocol=https;AccountName={};AccountKey={}'.\ format("testaccount", "wrongsecret") store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"Incorrect padding" in str(exc.value)
def test_wrong_endpoint(self): from azure.storage.common.retry import ExponentialRetry container = uuid() conn_string = create_azure_conn_string(load_azure_credentials()) conn_string += \ ";BlobEndpoint=https://hopenostorethere.blob.core.windows.net;" store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"Failed to establish a new connection" in str(exc.value)
def retries(self): # By default, retries are performed with an exponential backoff. # Any custom retry logic may be used by simply defining a retry function, # but several easy pre-written options are available with modifiable settings. client = TableService(account_name='<account_name>', account_key='<account_key>') # Use an exponential retry, but modify the backoff settings # Here, we increase the initial back off, increase the number of retry attempts # and decrease the base of the exponential backoff. client.retry = ExponentialRetry(initial_backoff=30, increment_power=2, max_attempts=5).retry # Use a default linear retry policy instead client.retry = LinearRetry().retry # Turn off retries client.retry = no_retry
def test_wrong_endpoint(self): container = str(uuid()) conn_string = create_azure_conn_string(load_azure_credentials()) conn_string += \ ";BlobEndpoint=https://hopenostorethere.blob.core.windows.net;" store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) if hasattr(store, 'block_blob_service'): from azure.storage.common.retry import ExponentialRetry store.block_blob_service.retry = ExponentialRetry( max_attempts=0).retry else: store.blob_container_client._config.retry_policy.total_retries = 0 with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"connect" in str(exc.value)
def create_client(stage_info, use_accelerate_endpoint=False): """ Creates a client object with a stage credential :param stage_credentials: a stage credential :param use_accelerate_endpoint: is accelerate endpoint? :return: client """ stage_credentials = stage_info[u'creds'] sas_token = stage_credentials[u'AZURE_SAS_TOKEN'] if sas_token and sas_token.startswith(u'?'): sas_token = sas_token[1:] client = BlockBlobService(account_name=stage_info[u'storageAccount'], sas_token=sas_token) client.retry = ExponentialRetry(initial_backoff=1, increment_base=2, max_attempts=60, random_jitter_range=2).retry return client
def test_retry_on_timeout(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry service.response_callback = ResponseCallback( status=201, new_status=408).override_status # Act try: created = service.create_container(container_name) finally: service.delete_container(container_name) # Assert # The initial create will return 201, but we overwrite it and retry. # The retry will then get a 409 and return false. self.assertFalse(created)
def test_invalid_account_key(self): # Arrange container_name = self.get_resource_name() service = BlockBlobService(account_name="dummy_account_name", account_key="dummy_account_key") # Shorten retries and add counter service.retry = ExponentialRetry(initial_backoff=1, increment_base=3, max_attempts=3).retry retry_counter = RetryCounter() service.retry_callback = retry_counter.simple_count # Act with self.assertRaises(AzureSigningError): service.get_container_metadata(container_name) # Assert # No retry should be performed since the signing error is fatal self.assertEqual(retry_counter.count, 0)
def test_wrong_endpoint(self): container = str(uuid()) conn_string = get_azure_conn_string() conn_settings = dict([s.split("=", 1) for s in conn_string.split(";") if s]) conn_settings['BlobEndpoint'] = 'https://host-does-not-exist/' conn_string = ';'.join('{}={}'.format(key, value) for key, value in conn_settings.items()) store = AzureBlockBlobStore(conn_string=conn_string, container=container, create_if_missing=False) if hasattr(store, 'block_blob_service'): from azure.storage.common.retry import ExponentialRetry store.block_blob_service.retry = ExponentialRetry( max_attempts=0 ).retry else: store.blob_container_client._config.retry_policy.total_retries = 0 with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"connect" in str(exc.value)
def test_retry_with_deserialization(self): # Arrange container_name = self.get_resource_name(prefix='retry') service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry try: created = service.create_container(container_name) # Act service.response_callback = ResponseCallback( status=200, new_status=408).override_first_status containers = service.list_containers(prefix='retry') finally: service.delete_container(container_name) # Assert self.assertTrue(len(list(containers)) >= 1)
def test_wrong_credentials(self): container = str(uuid()) conn_string = ( "DefaultEndpointsProtocol=https;AccountName={};AccountKey={}".format( "testaccount", "wrongsecret" ) ) store = AzureBlockBlobStore( conn_string=conn_string, container=container, create_if_missing=False ) if hasattr(store, "block_blob_service"): from azure.storage.common.retry import ExponentialRetry store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry else: store.blob_container_client._config.retry_policy.total_retries = 0 # type: ignore with pytest.raises(IOError) as exc: store.put(u"key", b"data") assert u"Incorrect padding" in str(exc.value)
def test_invalid_retry(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.retry = ExponentialRetry(initial_backoff=1, increment_base=2).retry # Force the create call to fail by pretending it's a teapot service.response_callback = ResponseCallback( status=201, new_status=418).override_status # Act try: service.create_container(container_name) self.fail('The callback should force failure.') except AzureHttpError as e: # Assert self.assertEqual(418, e.status_code) self.assertEqual('Created\n', e.args[0]) finally: service.delete_container(container_name)
def test_exponential_retry(self): # Arrange container_name = self.get_resource_name() service = self._create_storage_service(BlockBlobService, self.settings) service.create_container(container_name) service.retry = ExponentialRetry(max_attempts=3).retry # Force the create call to 'timeout' with a 408 response_callback = ResponseCallback(status=200, new_status=408) service.response_callback = response_callback.override_status # Act with self.assertRaises(AzureHttpError): service.get_container_metadata(container_name) # Assert the response was called the right number of times (1 initial request + 3 retries) self.assertEqual(response_callback.count, 1 + 3) # Clean up service.response_callback = None service.delete_container(container_name)
def _client(self): """Return the Azure Storage Block Blob service. If this is the first call to the property, the client is created and the container is created if it doesn't yet exist. """ client = BlockBlobService(connection_string=self._connection_string) created = client.create_container( container_name=self._container_name, fail_on_exist=False) if created: LOGGER.info("Created Azure Blob Storage container %s", self._container_name) client.retry = ExponentialRetry( initial_backoff=self._retry_initial_backoff_sec, increment_base=self._retry_increment_base, max_attempts=self._retry_max_attempts).retry return client
def read_from_secondary(self): # If you are using RA-GRS accounts, you may want to enable reading from the # secondary endpoint. Note that your application will have to handle this # data potentially being out of date as the secondary may be behind the # primary. client = TableService(account_name='<account_name>', account_key='<account_key>') # The location mode is set to primary by default meaning that all requests # are sent to the primary endpoint. If you'd like to instead read from the # secondary endpoint by default, set location mode to secondary. Note that # writes will continue to go to primary as they are not allowed on secondary. client.location_mode = LocationMode.SECONDARY # You may also decide you want to retry to secondary. This is useful if # you'd like to automatically handle the primary being temporarily down. # Again, your application will have to handle data being potentially out # of date. Retry to secondary logic may be built into a custom retry policy, # but our retry policies have a flag to enable it. Here we use the same # exponential retry as by default, but allow it to retry to secondary if # the initial request to primary fails. client.location_mode = LocationMode.PRIMARY # Reset the location_mode to start with primary client.retry = ExponentialRetry(retry_to_secondary=True).retry