def test_example_retry_policy():
url = "https://bing.com"
config = Configuration()
config.user_agent_policy = UserAgentPolicy("myusergant")
config.redirect_policy = RedirectPolicy()
# [START retry_policy]
from azure.core.pipeline.policies import RetryPolicy
config.retry_policy = RetryPolicy()
# Total number of retries to allow. Takes precedence over other counts.
# Default value is 10.
config.retry_policy.total_retries = 5
# How many connection-related errors to retry on.
# These are errors raised before the request is sent to the remote server,
# which we assume has not triggered the server to process the request. Default value is 3
config.retry_policy.connect_retries = 2
# How many times to retry on read errors.
# These errors are raised after the request was sent to the server, so the
# request may have side-effects. Default value is 3.
config.retry_policy.read_retries = 4
# How many times to retry on bad status codes. Default value is 3.
config.retry_policy.status_retries = 3
# A backoff factor to apply between attempts after the second try
# (most errors are resolved immediately by a second try without a delay).
# Retry policy will sleep for:
# {backoff factor} * (2 ** ({number of total retries} - 1))
# seconds. If the backoff_factor is 0.1, then the retry will sleep
# for [0.0s, 0.2s, 0.4s, ...] between retries.
# The default value is 0.8.
config.retry_policy.backoff_factor = 0.5
# The maximum back off time. Default value is 120 seconds (2 minutes).
config.retry_policy.backoff_max = 120
# Alternatively you can disable redirects entirely
config.retry_policy = RetryPolicy.no_retries()
# All of these settings can also be configured per operation.
client = PipelineClient(base_url=url, config=config)
request = client.get(url)
pipeline_response = client._pipeline.run(request,
retry_total=10,
retry_connect=1,
retry_read=1,
retry_status=5,
retry_backoff_factor=0.5,
retry_backoff_max=120,
retry_on_methods=['GET'])
# [END retry_policy]
response = pipeline_response.http_response
assert response.status_code == 200
def test_retry_seekable_body():
def build_response(body, content_type=None):
class MockResponse(HttpResponse):
def __init__(self):
super(MockResponse, self).__init__(None, None)
self._body = 'test'
def body(self):
return self._body
data = BytesIO(b"Lots of dataaaa")
universal_request = HttpRequest('GET', 'http://127.0.0.1/', data=data)
universal_request.set_streamed_data_body(data)
return PipelineResponse(universal_request, MockResponse(),
PipelineContext(None, stream=True))
response = build_response(b"<groot/>", content_type="application/xml")
http_retry = RetryPolicy()
setting = {
'total': 3,
'status': 3,
'history': [],
'connect': 3,
'read': 3,
'body_position': 10,
}
increment = http_retry.increment(setting, response)
assert increment
def _create_config(**kwargs):
# type: (Any) -> Configuration
config = Configuration(**kwargs)
config.logging_policy = NetworkTraceLoggingPolicy(**kwargs)
config.retry_policy = RetryPolicy(**kwargs)
config.proxy_policy = ProxyPolicy(**kwargs)
return config
def test_retry_on_429(http_request, http_response):
class MockTransport(HttpTransport):
def __init__(self):
self._count = 0
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def close(self):
pass
def open(self):
pass
def send(self, request,
**kwargs): # type: (PipelineRequest, Any) -> PipelineResponse
self._count += 1
response = create_http_response(http_response, request, None)
response.status_code = 429
return response
http_request = http_request('GET', 'http://localhost/')
http_retry = RetryPolicy(retry_total=1)
transport = MockTransport()
pipeline = Pipeline(transport, [http_retry])
pipeline.run(http_request)
assert transport._count == 2
def test_no_retry_on_201():
class MockTransport(HttpTransport):
def __init__(self):
self._count = 0
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def close(self):
pass
def open(self):
pass
def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse
self._count += 1
response = HttpResponse(request, None)
response.status_code = 201
headers = {"Retry-After": "1"}
response.headers = headers
return response
http_request = HttpRequest('GET', 'http://127.0.0.1/')
http_retry = RetryPolicy(retry_total = 1)
transport = MockTransport()
pipeline = Pipeline(transport, [http_retry])
pipeline.run(http_request)
assert transport._count == 1
def test_retry_timeout():
class MockTransport(HttpTransport):
def __init__(self):
self.count = 0
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def close(self):
pass
def open(self):
pass
def send(self, request,
**kwargs): # type: (PipelineRequest, Any) -> PipelineResponse
self.count += 1
if self.count > 2:
assert self.count <= 2
time.sleep(0.5)
raise ServiceResponseError('timeout')
http_request = HttpRequest('GET', 'http://127.0.0.1/')
headers = {'Content-Type': "multipart/form-data"}
http_request.headers = headers
http_retry = RetryPolicy(retry_total=10, timeout=1)
pipeline = Pipeline(MockTransport(), [http_retry])
with pytest.raises(ServiceResponseTimeoutError):
pipeline.run(http_request)
def test_x_ms_retry_after(retry_after_input):
retry_policy = RetryPolicy()
request = HttpRequest("GET", "https://bing.com")
response = HttpResponse(request, None)
response.headers["x-ms-retry-after-ms"] = retry_after_input
pipeline_response = PipelineResponse(request, response, None)
retry_after = retry_policy.get_retry_after(pipeline_response)
seconds = float(retry_after_input)
assert retry_after == seconds/1000.0
response.headers.pop("x-ms-retry-after-ms")
response.headers["Retry-After"] = retry_after_input
retry_after = retry_policy.get_retry_after(pipeline_response)
assert retry_after == float(retry_after_input)
response.headers["x-ms-retry-after-ms"] = 500
retry_after = retry_policy.get_retry_after(pipeline_response)
assert retry_after == float(retry_after_input)
def test_x_ms_retry_after(retry_after_input, http_request, http_response):
retry_policy = RetryPolicy()
request = http_request("GET", "http://localhost")
response = create_http_response(http_response, request, None)
response.headers["x-ms-retry-after-ms"] = retry_after_input
pipeline_response = PipelineResponse(request, response, None)
retry_after = retry_policy.get_retry_after(pipeline_response)
seconds = float(retry_after_input)
assert retry_after == seconds / 1000.0
response.headers.pop("x-ms-retry-after-ms")
response.headers["Retry-After"] = retry_after_input
retry_after = retry_policy.get_retry_after(pipeline_response)
assert retry_after == float(retry_after_input)
response.headers["x-ms-retry-after-ms"] = 500
retry_after = retry_policy.get_retry_after(pipeline_response)
assert retry_after == float(retry_after_input)
def test_retry_seekable_stream():
class MockTransport(HttpTransport):
def __init__(self):
self._first = True
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def close(self):
pass
def open(self):
pass
def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse
if self._first:
self._first = False
request.body.seek(0,2)
raise AzureError('fail on first')
position = request.body.tell()
assert position == 0
response = HttpResponse(request, None)
response.status_code = 400
return response
data = BytesIO(b"Lots of dataaaa")
http_request = HttpRequest('GET', 'http://127.0.0.1/')
http_request.set_streamed_data_body(data)
http_retry = RetryPolicy(retry_total = 1)
pipeline = Pipeline(MockTransport(), [http_retry])
pipeline.run(http_request)
def _create_config(**kwargs):
# type: (Mapping[str, Any]) -> Configuration
config = Configuration(**kwargs)
config.logging_policy = NetworkTraceLoggingPolicy(**kwargs)
config.retry_policy = RetryPolicy(**kwargs)
config.proxy_policy = ProxyPolicy(**kwargs)
config.user_agent_policy = UserAgentPolicy(base_user_agent=USER_AGENT, **kwargs)
return config
def create_config(**kwargs):
# type: (Mapping[str, Any]) -> Configuration
config = Configuration(**kwargs)
config.logging_policy = NetworkTraceLoggingPolicy(**kwargs)
config.retry_policy = RetryPolicy(retry_on_status_codes=[404, 429] +
list(range(500, 600)),
**kwargs)
return config
def test_retry_types():
history = ["1", "2", "3"]
settings = {'history': history, 'backoff': 1, 'max_backoff': 10}
retry_policy = RetryPolicy()
backoff_time = retry_policy.get_backoff_time(settings)
assert backoff_time == 4
retry_policy = RetryPolicy(retry_mode=RetryMode.Fixed)
backoff_time = retry_policy.get_backoff_time(settings)
assert backoff_time == 1
retry_policy = RetryPolicy(retry_mode=RetryMode.Exponential)
backoff_time = retry_policy.get_backoff_time(settings)
assert backoff_time == 4
def build_pipeline(transport=None, policies=None, **kwargs):
if not policies:
config = _get_config(**kwargs)
config.retry_policy = RetryPolicy(**kwargs)
policies = _get_policies(config, **kwargs)
if not transport:
transport = RequestsTransport(**kwargs)
return Pipeline(transport, policies=policies)
def test_retry_without_http_response():
class NaughtyPolicy(HTTPPolicy):
def send(*args):
raise AzureError('boo')
policies = [RetryPolicy(), NaughtyPolicy()]
pipeline = Pipeline(policies=policies, transport=None)
with pytest.raises(AzureError):
pipeline.run(HttpRequest('GET', url='https://foo.bar'))
def _create_configuration(self, **kwargs): # pylint: disable=no-self-use
config = Configuration(**kwargs)
config.user_agent_policy = kwargs.get("user_agent_policy") or UserAgentPolicy(**kwargs)
config.headers_policy = kwargs.get("headers_policy") or HeadersPolicy(**kwargs)
config.proxy_policy = kwargs.get("proxy_policy") or ProxyPolicy(**kwargs)
config.logging_policy = kwargs.get("logging_policy") or NetworkTraceLoggingPolicy(**kwargs)
config.retry_policy = kwargs.get("retry_policy") or RetryPolicy(**kwargs)
config.redirect_policy = kwargs.get("redirect_policy") or RedirectPolicy(**kwargs)
return config
def create_config(**kwargs):
# type: (Dict[str, str]) -> Configuration
timeout = kwargs.pop("connection_timeout", 2)
config = Configuration(connection_timeout=timeout, **kwargs)
config.logging_policy = NetworkTraceLoggingPolicy(**kwargs)
retries = kwargs.pop("retry_total", 5)
config.retry_policy = RetryPolicy(retry_total=retries,
retry_on_status_codes=[404, 429] +
list(range(500, 600)),
**kwargs)
return config
def create_config(credential, scopes, **kwargs):
# Here the SDK developer would define the default
# config to interact with the service
config = Configuration(**kwargs)
config.headers_policy = HeadersPolicy({"CustomHeader": "Value"}, **kwargs)
config.authentication_policy = BearerTokenCredentialPolicy(credential, scopes, **kwargs)
config.user_agent_policy = UserAgentPolicy("ServiceUserAgentValue", **kwargs)
config.retry_policy = RetryPolicy(**kwargs)
config.redirect_policy = RedirectPolicy(**kwargs)
config.logging_policy = NetworkTraceLoggingPolicy(**kwargs)
config.proxy_policy = ProxyPolicy(**kwargs)
def client(cookie_policy):
"""Create a AutoRestHttpInfrastructureTestService client with test server credentials."""
policies = [
HeadersPolicy(),
ContentDecodePolicy(),
RedirectPolicy(),
RetryPolicy(), cookie_policy
]
with AutoRestHttpInfrastructureTestService(
base_url="http://localhost:3000", policies=policies) as client:
yield client
def create_config(**kwargs):
# type: (Any) -> Configuration
config = Configuration(**kwargs)
config.user_agent_policy = UserAgentPolicy("KeyClient", **kwargs)
config.headers_policy = None
config.retry_policy = RetryPolicy(**kwargs)
config.redirect_policy = RedirectPolicy(**kwargs)
# TODO: these are requests-specific
config.cert = config.timeout = None
config.verify = True
return config
def test_example_no_retries():
url = "https://bing.com"
retry_policy = RetryPolicy.no_retries()
client = PipelineClient(base_url=url, policies=[retry_policy])
request = client.get(url)
pipeline_response = client._pipeline.run(request)
response = pipeline_response.http_response
# bing returns 301 if not retried
assert response.status_code == 301
def __init__(self, **options):
self._telemetry_processors = []
self.options = ExporterOptions(**options)
retry_policy = RetryPolicy(timeout=self.options.timeout)
proxy_policy = ProxyPolicy(proxies=self.options.proxies)
self.client = AzureMonitorClient(
self.options.endpoint, proxy_policy=proxy_policy, retry_policy=retry_policy)
self.storage = LocalFileStorage(
path=self.options.storage_path,
max_size=self.options.storage_max_size,
maintenance_period=self.options.storage_maintenance_period,
retention_period=self.options.storage_retention_period,
)
def _create_pipeline(**kwargs):
"""Creates and returns a PipelineClient configured for the provided base_url and kwargs"""
transport = kwargs.get("transport", RequestsTransport(**kwargs))
policies = [
kwargs.get("user_agent_policy",
UserAgentPolicy(_constants.USER_AGENT, **kwargs)),
kwargs.get("headers_policy", HeadersPolicy(**kwargs)),
kwargs.get("authentication_policy"),
kwargs.get("retry_policy", RetryPolicy(**kwargs)),
kwargs.get("redirect_policy", RedirectPolicy(**kwargs)),
kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)),
kwargs.get("proxy_policy", ProxyPolicy(**kwargs)),
]
return Pipeline(policies=policies, transport=transport)
def __init__(self, credential, **kwargs): # pylint:disable=super-init-not-called
self._set_universal(**kwargs)
# sync-specific azure pipeline policies
if isinstance(credential, str):
self.headers_policy.add_header("Ocp-Apim-Subscription-Key",
credential)
else:
scopes = kwargs.pop("scopes", [])
self.authentication_policy = BearerTokenCredentialPolicy(
credential, *scopes, **kwargs)
self.retry_policy = kwargs.get("retry_policy", RetryPolicy(**kwargs))
self.redirect_policy = kwargs.get("redirect_policy",
RedirectPolicy(**kwargs))
self.transport = kwargs.get("transport", RequestsTransport())
def test_retry_seekable_file(http_request, http_response):
class MockTransport(HttpTransport):
def __init__(self):
self._first = True
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def close(self):
pass
def open(self):
pass
def send(self, request,
**kwargs): # type: (PipelineRequest, Any) -> PipelineResponse
if self._first:
self._first = False
for value in request.files.values():
name, body = value[0], value[1]
if name and body and hasattr(body, 'read'):
body.seek(0, 2)
raise AzureError('fail on first')
for value in request.files.values():
name, body = value[0], value[1]
if name and body and hasattr(body, 'read'):
position = body.tell()
assert not position
response = create_http_response(http_response, request,
None)
response.status_code = 400
return response
file = tempfile.NamedTemporaryFile(delete=False)
file.write(b'Lots of dataaaa')
file.close()
http_request = http_request('GET', 'http://localhost/')
headers = {'Content-Type': "multipart/form-data"}
http_request.headers = headers
with open(file.name, 'rb') as f:
form_data_content = {
'fileContent': f,
'fileName': f.name,
}
http_request.set_formdata_body(form_data_content)
http_retry = RetryPolicy(retry_total=1)
pipeline = Pipeline(MockTransport(), [http_retry])
pipeline.run(http_request)
os.unlink(f.name)
def test_example_requests():
request = HttpRequest("GET", "https://bing.com")
policies = [
UserAgentPolicy("myuseragent"),
RedirectPolicy(),
RetryPolicy()
]
# [START requests]
from azure.core.pipeline.transport import RequestsTransport
with Pipeline(transport=RequestsTransport(), policies=policies) as pipeline:
response = pipeline.run(request)
# [END requests]
assert pipeline._transport.session is None
assert response.http_response.status_code == 200
def test_does_not_sleep_after_timeout(transport_error, expected_timeout_error):
# With default settings policy will sleep twice before exhausting its retries: 1.6s, 3.2s.
# It should not sleep the second time when given timeout=1
timeout = 1
transport = Mock(
spec=HttpTransport,
send=Mock(side_effect=transport_error("oops")),
sleep=Mock(wraps=time.sleep),
)
pipeline = Pipeline(transport, [RetryPolicy(timeout=timeout)])
with pytest.raises(expected_timeout_error):
pipeline.run(HttpRequest("GET", "http://127.0.0.1/"))
assert transport.sleep.call_count == 1
def test_retry_timeout():
timeout = 1
def send(request, **kwargs):
assert kwargs["connection_timeout"] <= timeout, "policy should set connection_timeout not to exceed timeout"
raise ServiceResponseError("oops")
transport = Mock(
spec=HttpTransport,
send=Mock(wraps=send),
connection_config=ConnectionConfiguration(connection_timeout=timeout * 2),
sleep=time.sleep,
)
pipeline = Pipeline(transport, [RetryPolicy(timeout=timeout)])
with pytest.raises(ServiceResponseTimeoutError):
response = pipeline.run(HttpRequest("GET", "http://127.0.0.1/"))
def test_retry_seekable_file():
class MockTransport(HttpTransport):
def __init__(self):
self._first = True
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def close(self):
pass
def open(self):
pass
def send(self, request,
**kwargs): # type: (PipelineRequest, Any) -> PipelineResponse
if self._first:
self._first = False
for value in request.files.values():
name, body = value[0], value[1]
if name and body and hasattr(body, 'read'):
body.seek(0, 2)
raise AzureError('fail on first')
for value in request.files.values():
name, body = value[0], value[1]
if name and body and hasattr(body, 'read'):
position = body.tell()
assert not position
return HttpResponse(request, None)
file_name = 'test_retry_seekable_file'
with open(file_name, "w+") as f:
f.write('Lots of dataaaa')
http_request = HttpRequest('GET', 'http://127.0.0.1/')
headers = {'Content-Type': "multipart/form-data"}
http_request.headers = headers
form_data_content = {
'fileContent': open(file_name, 'rb'),
'fileName': file_name,
}
http_request.set_formdata_body(form_data_content)
http_retry = RetryPolicy(retry_total=1)
pipeline = Pipeline(MockTransport(), [http_retry])
pipeline.run(http_request)
def test_timeout_defaults():
"""When "timeout" is not set, the policy should not override the transport's timeout configuration"""
def send(request, **kwargs):
for arg in ("connection_timeout", "read_timeout"):
assert arg not in kwargs, "policy should defer to transport configuration when not given a timeout"
response = HttpResponse(request, None)
response.status_code = 200
return response
transport = Mock(
spec_set=HttpTransport,
send=Mock(wraps=send),
sleep=Mock(side_effect=Exception("policy should not sleep: its first send succeeded")),
)
pipeline = Pipeline(transport, [RetryPolicy()])
pipeline.run(HttpRequest("GET", "http://127.0.0.1/"))
assert transport.send.call_count == 1, "policy should not retry: its first send succeeded"
def _policies(credential, **kwargs):
# type: (Union[AzureKeyCredential, EventGridSharedAccessSignatureCredential], Any) -> List[Any]
auth_policy = _get_authentication_policy(credential)
sdk_moniker = 'eventgrid/{}'.format(VERSION)
policies = [
RequestIdPolicy(**kwargs),
HeadersPolicy(**kwargs),
UserAgentPolicy(sdk_moniker=sdk_moniker, **kwargs),
ProxyPolicy(**kwargs),
ContentDecodePolicy(**kwargs),
RedirectPolicy(**kwargs),
RetryPolicy(**kwargs), auth_policy,
CustomHookPolicy(**kwargs),
NetworkTraceLoggingPolicy(**kwargs),
DistributedTracingPolicy(**kwargs),
CloudEventDistributedTracingPolicy(),
HttpLoggingPolicy(**kwargs)
]
return policies
