def test_exponential_retry_interval(self):
# Arrange
initial_backoff = 15
increment_power = 3
retry_policy = ExponentialRetry(initial_backoff, increment_power)
context_stub = RetryContext()
# Act
context_stub.count = 0
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 15)
# Act
context_stub.count = 1
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 18)
# Act
context_stub.count = 2
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 24)
# Act
context_stub.count = 3
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 42)
def create_client(stage_info, use_accelerate_endpoint=False):
"""
Creates a client object with a stage credential
:param stage_credentials: a stage credential
:param use_accelerate_endpoint: is accelerate endpoint?
:return: client
"""
stage_credentials = stage_info[u'creds']
sas_token = stage_credentials[u'AZURE_SAS_TOKEN']
if sas_token and sas_token.startswith(u'?'):
sas_token = sas_token[1:]
end_point = stage_info['endPoint']
if end_point.startswith('blob.'):
end_point = end_point[len('blob.'):]
client = BlockBlobService(account_name=stage_info[u'storageAccount'],
sas_token=sas_token,
endpoint_suffix=end_point)
client._httpclient = RawBodyReadingClient(session=requests.session(),
protocol="https",
timeout=2000)
client.retry = ExponentialRetry(initial_backoff=1,
increment_base=2,
max_attempts=60,
random_jitter_range=2).retry
return client
def test_secondary_location_mode(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.location_mode = LocationMode.SECONDARY
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
# Act
try:
service.create_container(container_name)
# Override the response from secondary if it's 404 as that simply means
# the container hasn't replicated. We're just testing we try secondary,
# so that's fine.
service.response_callback = ResponseCallback(
status=404, new_status=200).override_first_status
# Assert
def request_callback(request):
if self.settings.IS_EMULATED:
self.assertNotEqual(-1, request.path.find('-secondary'))
else:
self.assertNotEqual(-1, request.host.find('-secondary'))
service.request_callback = request_callback
service.get_container_metadata(container_name)
finally:
# Delete will go to primary, so disable the request validation
service.request_callback = None
service.delete_container(container_name)
def test_retry_to_secondary_with_put(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2).retry
# Act
try:
# Fail the first create attempt
service.response_callback = ResponseCallback(
status=201, new_status=408).override_first_status
# Assert
# Confirm that the create request does *not* get retried to secondary
# This should actually throw InvalidPermissions if sent to secondary,
# but validate the location_mode anyways.
def retry_callback(retry_context):
self.assertEqual(LocationMode.PRIMARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service.create_container(container_name)
finally:
service.response_callback = None
service.retry_callback = None
service.delete_container(container_name)
def test_retry_to_secondary_with_get(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2).retry
# Act
try:
service.create_container(container_name)
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
# Assert
# Confirm that the get request gets retried to secondary
def retry_callback(retry_context):
# Only check this every other time, sometimes the secondary location fails due to delay
if retry_context.count % 2 == 1:
self.assertEqual(LocationMode.SECONDARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service.get_container_metadata(container_name)
finally:
service.response_callback = None
service.retry_callback = None
service.delete_container(container_name)
def test_location_lock(self):
# Arrange
service = self._create_storage_service(BlockBlobService, self.settings)
# Act
# Fail the first request and set the retry policy to retry to secondary
service.retry = ExponentialRetry(retry_to_secondary=True,
initial_backoff=1,
increment_base=2).retry
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
context = _OperationContext(location_lock=True)
# Assert
# Confirm that the first request gets retried to secondary
# The given test account must be GRS
def retry_callback(retry_context):
self.assertEqual(LocationMode.SECONDARY,
retry_context.location_mode)
service.retry_callback = retry_callback
service._list_containers(prefix='lock', _context=context)
# Confirm that the second list request done with the same context sticks
# to the final location of the first list request (aka secondary) despite
# the client normally trying primary first
def request_callback(request):
if self.settings.IS_EMULATED:
self.assertNotEqual(-1, request.path.find('-secondary'))
else:
self.assertNotEqual(-1, request.host.find('-secondary'))
service.request_callback = request_callback
service._list_containers(prefix='lock', _context=context)
def __init__(self):
with open('./vardhamanbot/bot/config/app.json') as app_config_file:
app_config = json.load(app_config_file)
self.tableservice = TableService(app_config["ats_name"],
app_config["ats_key"])
self.tableservice.retry = ExponentialRetry(initial_backoff=30,
increment_base=2,
max_attempts=20).retry
def test_exponential_retry_interval(self):
# Arrange
retry_policy = ExponentialRetry(initial_backoff=1,
increment_base=3,
random_jitter_range=3)
context_stub = RetryContext()
for i in range(10):
# Act
context_stub.count = 0
backoff = retry_policy._backoff(context_stub)
# Assert backoff interval is within +/- 3 of 1
self.assertTrue(0 <= backoff <= 4)
# Act
context_stub.count = 1
backoff = retry_policy._backoff(context_stub)
# Assert backoff interval is within +/- 3 of 4(1+3^1)
self.assertTrue(1 <= backoff <= 7)
# Act
context_stub.count = 2
backoff = retry_policy._backoff(context_stub)
# Assert backoff interval is within +/- 3 of 10(1+3^2)
self.assertTrue(7 <= backoff <= 13)
# Act
context_stub.count = 3
backoff = retry_policy._backoff(context_stub)
# Assert backoff interval is within +/- 3 of 28(1+3^3)
self.assertTrue(25 <= backoff <= 31)
def test_exponential_retry_interval(self):
# Arrange
initial_backoff = 15
increment_power = 3
retry_policy = ExponentialRetry(initial_backoff, increment_power)
context_stub = RetryContext()
# Act
context_stub.count = 0
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 15)
# Act
context_stub.count = 1
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 18)
# Act
context_stub.count = 2
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 24)
# Act
context_stub.count = 3
backoff = retry_policy._backoff(context_stub)
# Assert
self.assertEqual(backoff, 42)
def test_wrong_credentials(self):
from azure.storage.common.retry import ExponentialRetry
container = uuid()
conn_string = \
'DefaultEndpointsProtocol=https;AccountName={};AccountKey={}'.\
format("testaccount", "wrongsecret")
store = AzureBlockBlobStore(conn_string=conn_string,
container=container,
create_if_missing=False)
store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry
with pytest.raises(IOError) as exc:
store.put(u"key", b"data")
assert u"Incorrect padding" in str(exc.value)
def test_wrong_endpoint(self):
from azure.storage.common.retry import ExponentialRetry
container = uuid()
conn_string = create_azure_conn_string(load_azure_credentials())
conn_string += \
";BlobEndpoint=https://hopenostorethere.blob.core.windows.net;"
store = AzureBlockBlobStore(conn_string=conn_string,
container=container,
create_if_missing=False)
store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry
with pytest.raises(IOError) as exc:
store.put(u"key", b"data")
assert u"Failed to establish a new connection" in str(exc.value)
def retries(self):
# By default, retries are performed with an exponential backoff.
# Any custom retry logic may be used by simply defining a retry function,
# but several easy pre-written options are available with modifiable settings.
client = TableService(account_name='<account_name>', account_key='<account_key>')
# Use an exponential retry, but modify the backoff settings
# Here, we increase the initial back off, increase the number of retry attempts
# and decrease the base of the exponential backoff.
client.retry = ExponentialRetry(initial_backoff=30, increment_power=2, max_attempts=5).retry
# Use a default linear retry policy instead
client.retry = LinearRetry().retry
# Turn off retries
client.retry = no_retry
def test_wrong_endpoint(self):
container = str(uuid())
conn_string = create_azure_conn_string(load_azure_credentials())
conn_string += \
";BlobEndpoint=https://hopenostorethere.blob.core.windows.net;"
store = AzureBlockBlobStore(conn_string=conn_string,
container=container,
create_if_missing=False)
if hasattr(store, 'block_blob_service'):
from azure.storage.common.retry import ExponentialRetry
store.block_blob_service.retry = ExponentialRetry(
max_attempts=0).retry
else:
store.blob_container_client._config.retry_policy.total_retries = 0
with pytest.raises(IOError) as exc:
store.put(u"key", b"data")
assert u"connect" in str(exc.value)
def create_client(stage_info, use_accelerate_endpoint=False):
"""
Creates a client object with a stage credential
:param stage_credentials: a stage credential
:param use_accelerate_endpoint: is accelerate endpoint?
:return: client
"""
stage_credentials = stage_info[u'creds']
sas_token = stage_credentials[u'AZURE_SAS_TOKEN']
if sas_token and sas_token.startswith(u'?'):
sas_token = sas_token[1:]
client = BlockBlobService(account_name=stage_info[u'storageAccount'],
sas_token=sas_token)
client.retry = ExponentialRetry(initial_backoff=1,
increment_base=2,
max_attempts=60,
random_jitter_range=2).retry
return client
def test_retry_on_timeout(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
service.response_callback = ResponseCallback(
status=201, new_status=408).override_status
# Act
try:
created = service.create_container(container_name)
finally:
service.delete_container(container_name)
# Assert
# The initial create will return 201, but we overwrite it and retry.
# The retry will then get a 409 and return false.
self.assertFalse(created)
def test_invalid_account_key(self):
# Arrange
container_name = self.get_resource_name()
service = BlockBlobService(account_name="dummy_account_name",
account_key="dummy_account_key")
# Shorten retries and add counter
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=3,
max_attempts=3).retry
retry_counter = RetryCounter()
service.retry_callback = retry_counter.simple_count
# Act
with self.assertRaises(AzureSigningError):
service.get_container_metadata(container_name)
# Assert
# No retry should be performed since the signing error is fatal
self.assertEqual(retry_counter.count, 0)
def test_wrong_endpoint(self):
container = str(uuid())
conn_string = get_azure_conn_string()
conn_settings = dict([s.split("=", 1) for s in conn_string.split(";") if s])
conn_settings['BlobEndpoint'] = 'https://host-does-not-exist/'
conn_string = ';'.join('{}={}'.format(key, value) for key, value in conn_settings.items())
store = AzureBlockBlobStore(conn_string=conn_string,
container=container,
create_if_missing=False)
if hasattr(store, 'block_blob_service'):
from azure.storage.common.retry import ExponentialRetry
store.block_blob_service.retry = ExponentialRetry(
max_attempts=0
).retry
else:
store.blob_container_client._config.retry_policy.total_retries = 0
with pytest.raises(IOError) as exc:
store.put(u"key", b"data")
assert u"connect" in str(exc.value)
def test_retry_with_deserialization(self):
# Arrange
container_name = self.get_resource_name(prefix='retry')
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
try:
created = service.create_container(container_name)
# Act
service.response_callback = ResponseCallback(
status=200, new_status=408).override_first_status
containers = service.list_containers(prefix='retry')
finally:
service.delete_container(container_name)
# Assert
self.assertTrue(len(list(containers)) >= 1)
def test_wrong_credentials(self):
container = str(uuid())
conn_string = (
"DefaultEndpointsProtocol=https;AccountName={};AccountKey={}".format(
"testaccount", "wrongsecret"
)
)
store = AzureBlockBlobStore(
conn_string=conn_string, container=container, create_if_missing=False
)
if hasattr(store, "block_blob_service"):
from azure.storage.common.retry import ExponentialRetry
store.block_blob_service.retry = ExponentialRetry(max_attempts=0).retry
else:
store.blob_container_client._config.retry_policy.total_retries = 0 # type: ignore
with pytest.raises(IOError) as exc:
store.put(u"key", b"data")
assert u"Incorrect padding" in str(exc.value)
def test_invalid_retry(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.retry = ExponentialRetry(initial_backoff=1,
increment_base=2).retry
# Force the create call to fail by pretending it's a teapot
service.response_callback = ResponseCallback(
status=201, new_status=418).override_status
# Act
try:
service.create_container(container_name)
self.fail('The callback should force failure.')
except AzureHttpError as e:
# Assert
self.assertEqual(418, e.status_code)
self.assertEqual('Created\n', e.args[0])
finally:
service.delete_container(container_name)
def test_exponential_retry(self):
# Arrange
container_name = self.get_resource_name()
service = self._create_storage_service(BlockBlobService, self.settings)
service.create_container(container_name)
service.retry = ExponentialRetry(max_attempts=3).retry
# Force the create call to 'timeout' with a 408
response_callback = ResponseCallback(status=200, new_status=408)
service.response_callback = response_callback.override_status
# Act
with self.assertRaises(AzureHttpError):
service.get_container_metadata(container_name)
# Assert the response was called the right number of times (1 initial request + 3 retries)
self.assertEqual(response_callback.count, 1 + 3)
# Clean up
service.response_callback = None
service.delete_container(container_name)
def _client(self):
"""Return the Azure Storage Block Blob service.
If this is the first call to the property, the client is created and
the container is created if it doesn't yet exist.
"""
client = BlockBlobService(connection_string=self._connection_string)
created = client.create_container(
container_name=self._container_name, fail_on_exist=False)
if created:
LOGGER.info("Created Azure Blob Storage container %s",
self._container_name)
client.retry = ExponentialRetry(
initial_backoff=self._retry_initial_backoff_sec,
increment_base=self._retry_increment_base,
max_attempts=self._retry_max_attempts).retry
return client
def read_from_secondary(self):
# If you are using RA-GRS accounts, you may want to enable reading from the
# secondary endpoint. Note that your application will have to handle this
# data potentially being out of date as the secondary may be behind the
# primary.
client = TableService(account_name='<account_name>', account_key='<account_key>')
# The location mode is set to primary by default meaning that all requests
# are sent to the primary endpoint. If you'd like to instead read from the
# secondary endpoint by default, set location mode to secondary. Note that
# writes will continue to go to primary as they are not allowed on secondary.
client.location_mode = LocationMode.SECONDARY
# You may also decide you want to retry to secondary. This is useful if
# you'd like to automatically handle the primary being temporarily down.
# Again, your application will have to handle data being potentially out
# of date. Retry to secondary logic may be built into a custom retry policy,
# but our retry policies have a flag to enable it. Here we use the same
# exponential retry as by default, but allow it to retry to secondary if
# the initial request to primary fails.
client.location_mode = LocationMode.PRIMARY # Reset the location_mode to start with primary
client.retry = ExponentialRetry(retry_to_secondary=True).retry