def test_retry_types(): history = ["1", "2", "3"] settings = {'history': history, 'backoff': 1, 'max_backoff': 10} retry_policy = AsyncRetryPolicy() backoff_time = retry_policy.get_backoff_time(settings) assert backoff_time == 4 retry_policy = AsyncRetryPolicy(retry_mode=RetryMode.Fixed) backoff_time = retry_policy.get_backoff_time(settings) assert backoff_time == 1 retry_policy = AsyncRetryPolicy(retry_mode=RetryMode.Exponential) backoff_time = retry_policy.get_backoff_time(settings) assert backoff_time == 4
async def test_no_retry_on_201(http_request): class MockTransport(AsyncHttpTransport): def __init__(self): self._count = 0 async def __aexit__(self, exc_type, exc_val, exc_tb): pass async def close(self): pass async def open(self): pass async def send( self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse self._count += 1 response = HttpResponse(request, None) response.status_code = 201 headers = {"Retry-After": "1"} response.headers = headers return response http_request = http_request('GET', 'http://localhost/') http_retry = AsyncRetryPolicy(retry_total=1) transport = MockTransport() pipeline = AsyncPipeline(transport, [http_retry]) await pipeline.run(http_request) assert transport._count == 1
def _create_config(**kwargs: "Any") -> Configuration: config = Configuration(**kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) config.retry_policy = AsyncRetryPolicy(**kwargs) config.proxy_policy = ProxyPolicy(**kwargs) config.user_agent_policy = UserAgentPolicy(base_user_agent=USER_AGENT, **kwargs) return config
async def test_retry_timeout(): class MockTransport(AsyncHttpTransport): def __init__(self): self.count = 0 async def __aexit__(self, exc_type, exc_val, exc_tb): pass async def close(self): pass async def open(self): pass async def send( self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse self.count += 1 if self.count > 2: assert self.count <= 2 time.sleep(0.5) raise ServiceResponseError('timeout') http_request = HttpRequest('GET', 'http://127.0.0.1/') headers = {'Content-Type': "multipart/form-data"} http_request.headers = headers http_retry = AsyncRetryPolicy(retry_total=10, timeout=1) pipeline = AsyncPipeline(MockTransport(), [http_retry]) with pytest.raises(ServiceResponseTimeoutError): await pipeline.run(http_request)
async def test_example_async_retry_policy(): url = "https://bing.com" request = HttpRequest("GET", "https://bing.com") policies = [ UserAgentPolicy("myuseragent"), AsyncRedirectPolicy(), ] # [START async_retry_policy] from azure.core.pipeline.policies import AsyncRetryPolicy retry_policy = AsyncRetryPolicy() # Total number of retries to allow. Takes precedence over other counts. # Default value is 10. retry_policy.total_retries = 5 # How many connection-related errors to retry on. # These are errors raised before the request is sent to the remote server, # which we assume has not triggered the server to process the request. Default value is 3 retry_policy.connect_retries = 2 # How many times to retry on read errors. # These errors are raised after the request was sent to the server, so the # request may have side-effects. Default value is 3. retry_policy.read_retries = 4 # How many times to retry on bad status codes. Default value is 3. retry_policy.status_retries = 3 # A backoff factor to apply between attempts after the second try # (most errors are resolved immediately by a second try without a delay). # Retry policy will sleep for: # {backoff factor} * (2 ** ({number of total retries} - 1)) # seconds. If the backoff_factor is 0.1, then the retry will sleep # for [0.0s, 0.2s, 0.4s, ...] between retries. # The default value is 0.8. retry_policy.backoff_factor = 0.5 # The maximum back off time. Default value is 120 seconds (2 minutes). retry_policy.backoff_max = 120 # Alternatively you can disable redirects entirely retry_policy = AsyncRetryPolicy.no_retries() # All of these settings can also be configured per operation. policies.append(retry_policy) async with AsyncPipelineClient(base_url=url, policies=policies) as client: response = await client._pipeline.run(request, retry_total=10, retry_connect=1, retry_read=1, retry_status=5, retry_backoff_factor=0.5, retry_backoff_max=60, retry_on_methods=['GET']) # [END async_retry_policy] assert client._pipeline._transport.session is None assert response.http_response.status_code == 200
def _create_pipeline(self, credential, **kwargs): credential_policy = None if credential is None: raise ValueError("Parameter 'credential' must not be None.") if hasattr(credential, "get_token"): credential_policy = AsyncBearerTokenCredentialPolicy( credential, "https://cognitiveservices.azure.com/.default") elif isinstance(credential, six.string_types): credential_policy = CognitiveServicesCredentialPolicy(credential) elif credential is not None: raise TypeError("Unsupported credential: {}".format(credential)) config = self._create_configuration(**kwargs) config.transport = kwargs.get("transport") # type: ignore if not config.transport: try: from azure.core.pipeline.transport import AioHttpTransport except ImportError: raise ImportError( "Unable to create async transport. Please check aiohttp is installed." ) config.transport = AioHttpTransport(**kwargs) policies = [ config.headers_policy, config.user_agent_policy, RequestIdPolicy(**kwargs), config.proxy_policy, AsyncRedirectPolicy(**kwargs), AsyncRetryPolicy(**kwargs), credential_policy, config.logging_policy, AsyncTextAnalyticsResponseHook(**kwargs), DistributedTracingPolicy(**kwargs), HttpLoggingPolicy(**kwargs) ] return AsyncPipeline(config.transport, policies=policies)
async def test_retry_seekable_stream(): class MockTransport(AsyncHttpTransport): def __init__(self): self._first = True async def __aexit__(self, exc_type, exc_val, exc_tb): pass async def close(self): pass async def open(self): pass async def send( self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse if self._first: self._first = False request.body.seek(0, 2) raise AzureError('fail on first') position = request.body.tell() assert position == 0 response = HttpResponse(request, None) response.status_code = 400 return response data = BytesIO(b"Lots of dataaaa") http_request = HttpRequest('GET', 'http://127.0.0.1/') http_request.set_streamed_data_body(data) http_retry = AsyncRetryPolicy(retry_total=1) pipeline = AsyncPipeline(MockTransport(), [http_retry]) await pipeline.run(http_request)
async def test_retry_on_429(): class MockTransport(AsyncHttpTransport): def __init__(self): self._count = 0 async def __aexit__(self, exc_type, exc_val, exc_tb): pass async def close(self): pass async def open(self): pass async def send( self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse self._count += 1 response = HttpResponse(request, None) response.status_code = 429 return response http_request = HttpRequest('GET', 'http://127.0.0.1/') http_retry = AsyncRetryPolicy(retry_total=1) transport = MockTransport() pipeline = AsyncPipeline(transport, [http_retry]) await pipeline.run(http_request) assert transport._count == 2
async def test_retry_without_http_response(): class NaughtyPolicy(AsyncHTTPPolicy): def send(*args): raise AzureError('boo') policies = [AsyncRetryPolicy(), NaughtyPolicy()] pipeline = AsyncPipeline(policies=policies, transport=None) with pytest.raises(AzureError): await pipeline.run(HttpRequest('GET', url='https://foo.bar'))
def create_config(**kwargs: Dict[str, Any]) -> Configuration: timeout = kwargs.pop("connection_timeout", 2) config = Configuration(connection_timeout=timeout, **kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) retries = kwargs.pop("retry_total", 5) config.retry_policy = AsyncRetryPolicy( retry_total=retries, retry_on_status_codes=[404, 429] + list(range(500, 600)), **kwargs) return config
async def client(cookie_policy): """Create a AutoRestHttpInfrastructureTestService client with test server credentials.""" policies = [ HeadersPolicy(), ContentDecodePolicy(), AsyncRedirectPolicy(), AsyncRetryPolicy(), cookie_policy ] async with AutoRestHttpInfrastructureTestService(base_url="http://localhost:3000", policies=policies) as client: await yield_(client)
async def test_retry_seekable_file(): class MockTransport(AsyncHttpTransport): def __init__(self): self._first = True async def __aexit__(self, exc_type, exc_val, exc_tb): pass async def close(self): pass async def open(self): pass async def send( self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse if self._first: self._first = False for value in request.files.values(): name, body = value[0], value[1] if name and body and hasattr(body, 'read'): body.seek(0, 2) raise AzureError('fail on first') for value in request.files.values(): name, body = value[0], value[1] if name and body and hasattr(body, 'read'): position = body.tell() assert not position response = HttpResponse(request, None) response.status_code = 400 return response file = tempfile.NamedTemporaryFile(delete=False) file.write(b'Lots of dataaaa') file.close() http_request = HttpRequest('GET', 'http://127.0.0.1/') headers = {'Content-Type': "multipart/form-data"} http_request.headers = headers with open(file.name, 'rb') as f: form_data_content = { 'fileContent': f, 'fileName': f.name, } http_request.set_formdata_body(form_data_content) http_retry = AsyncRetryPolicy(retry_total=1) pipeline = AsyncPipeline(MockTransport(), [http_retry]) await pipeline.run(http_request) os.unlink(f.name)
def build_async_pipeline(transport=None, policies=None, **kwargs): from azure.core.pipeline import AsyncPipeline if not policies: from azure.core.pipeline.policies import AsyncRetryPolicy config = _get_config(**kwargs) config.retry_policy = AsyncRetryPolicy(**kwargs) policies = _get_policies(config, **kwargs) if not transport: from azure.core.pipeline.transport import AioHttpTransport transport = AioHttpTransport(**kwargs) return AsyncPipeline(transport, policies=policies)
def __init__(self, credential, **kwargs): # pylint:disable=super-init-not-called self._set_universal(**kwargs) # async-specific azure pipeline policies if isinstance(credential, str): self.headers_policy.add_header("Ocp-Apim-Subscription-Key", credential) else: scopes = kwargs.pop("scopes", []) self.authentication_policy = AsyncBearerTokenCredentialPolicy( credential, *scopes, **kwargs) self.retry_policy = kwargs.get("retry_policy", AsyncRetryPolicy(**kwargs)) self.redirect_policy = kwargs.get("redirect_policy", AsyncRedirectPolicy(**kwargs)) self.transport = kwargs.get("transport", AioHttpTransport())
def test_retry_after(retry_after_input, http_request): retry_policy = AsyncRetryPolicy() request = http_request("GET", "http://localhost") response = HttpResponse(request, None) response.headers["retry-after-ms"] = retry_after_input pipeline_response = PipelineResponse(request, response, None) retry_after = retry_policy.get_retry_after(pipeline_response) seconds = float(retry_after_input) assert retry_after == seconds / 1000.0 response.headers.pop("retry-after-ms") response.headers["Retry-After"] = retry_after_input retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input) response.headers["retry-after-ms"] = 500 retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input)
def test_x_ms_retry_after(retry_after_input): retry_policy = AsyncRetryPolicy() request = HttpRequest("GET", "https://bing.com") response = HttpResponse(request, None) response.headers["x-ms-retry-after-ms"] = retry_after_input pipeline_response = PipelineResponse(request, response, None) retry_after = retry_policy.get_retry_after(pipeline_response) seconds = float(retry_after_input) assert retry_after == seconds / 1000.0 response.headers.pop("x-ms-retry-after-ms") response.headers["Retry-After"] = retry_after_input retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input) response.headers["x-ms-retry-after-ms"] = 500 retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input)
async def test_does_not_sleep_after_timeout(transport_error, expected_timeout_error): # With default settings policy will sleep twice before exhausting its retries: 1.6s, 3.2s. # It should not sleep the second time when given timeout=1 timeout = 1 transport = Mock( spec=AsyncHttpTransport, send=Mock(side_effect=transport_error("oops")), sleep=Mock(wraps=asyncio.sleep), ) pipeline = AsyncPipeline(transport, [AsyncRetryPolicy(timeout=timeout)]) with pytest.raises(expected_timeout_error): await pipeline.run(HttpRequest("GET", "http://localhost/")) assert transport.sleep.call_count == 1
async def test_cloud_shell_live(cloud_shell): credential = ManagedIdentityCredential() token = credential.get_token("https://vault.azure.net") # Validate the token by sending a request to the Key Vault. The request is manual because azure-keyvault-secrets # can't authenticate in Cloud Shell; the MSI endpoint there doesn't support AADv2 scopes. policies = [ ContentDecodePolicy(), AsyncRedirectPolicy(), AsyncRetryPolicy(), HttpLoggingPolicy() ] client = AsyncPipelineClient(cloud_shell["vault_url"], policies=policies) list_secrets = client.get( "secrets", headers={"Authorization": "Bearer " + token.token}, params={"api-version": "7.0"}) async with client: await client._pipeline.run(list_secrets)
def _policies(credential: Union[AzureKeyCredential, AzureSasCredential], **kwargs: Any) -> List[Any]: auth_policy = _get_authentication_policy(credential) sdk_moniker = 'eventgridpublisherclient/{}'.format(VERSION) policies = [ RequestIdPolicy(**kwargs), HeadersPolicy(**kwargs), UserAgentPolicy(sdk_moniker=sdk_moniker, **kwargs), ProxyPolicy(**kwargs), ContentDecodePolicy(**kwargs), AsyncRedirectPolicy(**kwargs), AsyncRetryPolicy(**kwargs), auth_policy, CustomHookPolicy(**kwargs), NetworkTraceLoggingPolicy(**kwargs), DistributedTracingPolicy(**kwargs), CloudEventDistributedTracingPolicy(), HttpLoggingPolicy(**kwargs) ] return policies
async def test_retry_timeout(): timeout = 1 def send(request, **kwargs): assert kwargs[ "connection_timeout"] <= timeout, "policy should set connection_timeout not to exceed timeout" raise ServiceResponseError("oops") transport = Mock( spec=AsyncHttpTransport, send=Mock(wraps=send), connection_config=ConnectionConfiguration(connection_timeout=timeout * 2), sleep=asyncio.sleep, ) pipeline = AsyncPipeline(transport, [AsyncRetryPolicy(timeout=timeout)]) with pytest.raises(ServiceResponseTimeoutError): await pipeline.run(HttpRequest("GET", "http://127.0.0.1/"))
async def test_timeout_defaults(): """When "timeout" is not set, the policy should not override the transport's timeout configuration""" async def send(request, **kwargs): for arg in ("connection_timeout", "read_timeout"): assert arg not in kwargs, "policy should defer to transport configuration when not given a timeout" response = HttpResponse(request, None) response.status_code = 200 return response transport = Mock( spec_set=AsyncHttpTransport, send=Mock(wraps=send), sleep=Mock(side_effect=Exception( "policy should not sleep: its first send succeeded")), ) pipeline = AsyncPipeline(transport, [AsyncRetryPolicy()]) await pipeline.run(HttpRequest("GET", "http://127.0.0.1/")) assert transport.send.call_count == 1, "policy should not retry: its first send succeeded"
def _create_config(**kwargs: "Any") -> Configuration: config = Configuration(**kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) config.retry_policy = AsyncRetryPolicy(**kwargs) config.proxy_policy = ProxyPolicy(**kwargs) return config
async def test_add_custom_policy(): class BooPolicy(AsyncHTTPPolicy): def send(*args): raise AzureError('boo') class FooPolicy(AsyncHTTPPolicy): def send(*args): raise AzureError('boo') config = Configuration() retry_policy = AsyncRetryPolicy() config.retry_policy = retry_policy boo_policy = BooPolicy() foo_policy = FooPolicy() client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=boo_policy) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=[boo_policy]) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_retry_policies=boo_policy) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo > pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_retry_policies=[boo_policy]) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo > pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=boo_policy, per_retry_policies=foo_policy) policies = client._pipeline._impl_policies assert boo_policy in policies assert foo_policy in policies pos_boo = policies.index(boo_policy) pos_foo = policies.index(foo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry assert pos_foo > pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=[boo_policy], per_retry_policies=[foo_policy]) policies = client._pipeline._impl_policies assert boo_policy in policies assert foo_policy in policies pos_boo = policies.index(boo_policy) pos_foo = policies.index(foo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry assert pos_foo > pos_retry policies = [ UserAgentPolicy(), AsyncRetryPolicy(), DistributedTracingPolicy() ] client = AsyncPipelineClient(base_url="test", policies=policies, per_call_policies=boo_policy) actual_policies = client._pipeline._impl_policies assert boo_policy == actual_policies[0] client = AsyncPipelineClient(base_url="test", policies=policies, per_call_policies=[boo_policy]) actual_policies = client._pipeline._impl_policies assert boo_policy == actual_policies[0] client = AsyncPipelineClient(base_url="test", policies=policies, per_retry_policies=foo_policy) actual_policies = client._pipeline._impl_policies assert foo_policy == actual_policies[2] client = AsyncPipelineClient(base_url="test", policies=policies, per_retry_policies=[foo_policy]) actual_policies = client._pipeline._impl_policies assert foo_policy == actual_policies[2] client = AsyncPipelineClient(base_url="test", policies=policies, per_call_policies=boo_policy, per_retry_policies=[foo_policy]) actual_policies = client._pipeline._impl_policies assert boo_policy == actual_policies[0] assert foo_policy == actual_policies[3] client = AsyncPipelineClient(base_url="test", policies=policies, per_call_policies=[boo_policy], per_retry_policies=[foo_policy]) actual_policies = client._pipeline._impl_policies assert boo_policy == actual_policies[0] assert foo_policy == actual_policies[3] policies = [UserAgentPolicy(), DistributedTracingPolicy()] with pytest.raises(ValueError): client = AsyncPipelineClient(base_url="test", policies=policies, per_retry_policies=foo_policy) with pytest.raises(ValueError): client = AsyncPipelineClient(base_url="test", policies=policies, per_retry_policies=[foo_policy])
def _create_config(**kwargs: Mapping[str, Any]) -> Configuration: config = Configuration(**kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) config.retry_policy = AsyncRetryPolicy(**kwargs) return config
async def test_add_custom_policy(): class BooPolicy(AsyncHTTPPolicy): def send(*args): raise AzureError('boo') class FooPolicy(AsyncHTTPPolicy): def send(*args): raise AzureError('boo') config = Configuration() retry_policy = AsyncRetryPolicy() config.retry_policy = retry_policy boo_policy = BooPolicy() foo_policy = FooPolicy() client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=boo_policy) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=[boo_policy]) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_retry_policies=boo_policy) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo > pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_retry_policies=[boo_policy]) policies = client._pipeline._impl_policies assert boo_policy in policies pos_boo = policies.index(boo_policy) pos_retry = policies.index(retry_policy) assert pos_boo > pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=boo_policy, per_retry_policies=foo_policy) policies = client._pipeline._impl_policies assert boo_policy in policies assert foo_policy in policies pos_boo = policies.index(boo_policy) pos_foo = policies.index(foo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry assert pos_foo > pos_retry client = AsyncPipelineClient(base_url="test", config=config, per_call_policies=[boo_policy], per_retry_policies=[foo_policy]) policies = client._pipeline._impl_policies assert boo_policy in policies assert foo_policy in policies pos_boo = policies.index(boo_policy) pos_foo = policies.index(foo_policy) pos_retry = policies.index(retry_policy) assert pos_boo < pos_retry assert pos_foo > pos_retry
def _get_configuration(**kwargs: "Any") -> Configuration: config = Configuration() config.retry_policy = AsyncRetryPolicy(**kwargs) return config
def test_retry_code_class_variables(): retry_policy = AsyncRetryPolicy() assert retry_policy._RETRY_CODES is not None assert 408 in retry_policy._RETRY_CODES assert 429 in retry_policy._RETRY_CODES assert 501 not in retry_policy._RETRY_CODES