def test_example_retry_policy(): url = "https://bing.com" config = Configuration() config.user_agent_policy = UserAgentPolicy("myusergant") config.redirect_policy = RedirectPolicy() # [START retry_policy] from azure.core.pipeline.policies import RetryPolicy config.retry_policy = RetryPolicy() # Total number of retries to allow. Takes precedence over other counts. # Default value is 10. config.retry_policy.total_retries = 5 # How many connection-related errors to retry on. # These are errors raised before the request is sent to the remote server, # which we assume has not triggered the server to process the request. Default value is 3 config.retry_policy.connect_retries = 2 # How many times to retry on read errors. # These errors are raised after the request was sent to the server, so the # request may have side-effects. Default value is 3. config.retry_policy.read_retries = 4 # How many times to retry on bad status codes. Default value is 3. config.retry_policy.status_retries = 3 # A backoff factor to apply between attempts after the second try # (most errors are resolved immediately by a second try without a delay). # Retry policy will sleep for: # {backoff factor} * (2 ** ({number of total retries} - 1)) # seconds. If the backoff_factor is 0.1, then the retry will sleep # for [0.0s, 0.2s, 0.4s, ...] between retries. # The default value is 0.8. config.retry_policy.backoff_factor = 0.5 # The maximum back off time. Default value is 120 seconds (2 minutes). config.retry_policy.backoff_max = 120 # Alternatively you can disable redirects entirely config.retry_policy = RetryPolicy.no_retries() # All of these settings can also be configured per operation. client = PipelineClient(base_url=url, config=config) request = client.get(url) pipeline_response = client._pipeline.run(request, retry_total=10, retry_connect=1, retry_read=1, retry_status=5, retry_backoff_factor=0.5, retry_backoff_max=120, retry_on_methods=['GET']) # [END retry_policy] response = pipeline_response.http_response assert response.status_code == 200
def test_retry_seekable_body(): def build_response(body, content_type=None): class MockResponse(HttpResponse): def __init__(self): super(MockResponse, self).__init__(None, None) self._body = 'test' def body(self): return self._body data = BytesIO(b"Lots of dataaaa") universal_request = HttpRequest('GET', 'http://127.0.0.1/', data=data) universal_request.set_streamed_data_body(data) return PipelineResponse(universal_request, MockResponse(), PipelineContext(None, stream=True)) response = build_response(b"<groot/>", content_type="application/xml") http_retry = RetryPolicy() setting = { 'total': 3, 'status': 3, 'history': [], 'connect': 3, 'read': 3, 'body_position': 10, } increment = http_retry.increment(setting, response) assert increment
def _create_config(**kwargs): # type: (Any) -> Configuration config = Configuration(**kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) config.retry_policy = RetryPolicy(**kwargs) config.proxy_policy = ProxyPolicy(**kwargs) return config
def test_retry_on_429(http_request, http_response): class MockTransport(HttpTransport): def __init__(self): self._count = 0 def __exit__(self, exc_type, exc_val, exc_tb): pass def close(self): pass def open(self): pass def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse self._count += 1 response = create_http_response(http_response, request, None) response.status_code = 429 return response http_request = http_request('GET', 'http://localhost/') http_retry = RetryPolicy(retry_total=1) transport = MockTransport() pipeline = Pipeline(transport, [http_retry]) pipeline.run(http_request) assert transport._count == 2
def test_no_retry_on_201(): class MockTransport(HttpTransport): def __init__(self): self._count = 0 def __exit__(self, exc_type, exc_val, exc_tb): pass def close(self): pass def open(self): pass def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse self._count += 1 response = HttpResponse(request, None) response.status_code = 201 headers = {"Retry-After": "1"} response.headers = headers return response http_request = HttpRequest('GET', 'http://127.0.0.1/') http_retry = RetryPolicy(retry_total = 1) transport = MockTransport() pipeline = Pipeline(transport, [http_retry]) pipeline.run(http_request) assert transport._count == 1
def test_retry_timeout(): class MockTransport(HttpTransport): def __init__(self): self.count = 0 def __exit__(self, exc_type, exc_val, exc_tb): pass def close(self): pass def open(self): pass def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse self.count += 1 if self.count > 2: assert self.count <= 2 time.sleep(0.5) raise ServiceResponseError('timeout') http_request = HttpRequest('GET', 'http://127.0.0.1/') headers = {'Content-Type': "multipart/form-data"} http_request.headers = headers http_retry = RetryPolicy(retry_total=10, timeout=1) pipeline = Pipeline(MockTransport(), [http_retry]) with pytest.raises(ServiceResponseTimeoutError): pipeline.run(http_request)
def test_x_ms_retry_after(retry_after_input): retry_policy = RetryPolicy() request = HttpRequest("GET", "https://bing.com") response = HttpResponse(request, None) response.headers["x-ms-retry-after-ms"] = retry_after_input pipeline_response = PipelineResponse(request, response, None) retry_after = retry_policy.get_retry_after(pipeline_response) seconds = float(retry_after_input) assert retry_after == seconds/1000.0 response.headers.pop("x-ms-retry-after-ms") response.headers["Retry-After"] = retry_after_input retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input) response.headers["x-ms-retry-after-ms"] = 500 retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input)
def test_x_ms_retry_after(retry_after_input, http_request, http_response): retry_policy = RetryPolicy() request = http_request("GET", "http://localhost") response = create_http_response(http_response, request, None) response.headers["x-ms-retry-after-ms"] = retry_after_input pipeline_response = PipelineResponse(request, response, None) retry_after = retry_policy.get_retry_after(pipeline_response) seconds = float(retry_after_input) assert retry_after == seconds / 1000.0 response.headers.pop("x-ms-retry-after-ms") response.headers["Retry-After"] = retry_after_input retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input) response.headers["x-ms-retry-after-ms"] = 500 retry_after = retry_policy.get_retry_after(pipeline_response) assert retry_after == float(retry_after_input)
def test_retry_seekable_stream(): class MockTransport(HttpTransport): def __init__(self): self._first = True def __exit__(self, exc_type, exc_val, exc_tb): pass def close(self): pass def open(self): pass def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse if self._first: self._first = False request.body.seek(0,2) raise AzureError('fail on first') position = request.body.tell() assert position == 0 response = HttpResponse(request, None) response.status_code = 400 return response data = BytesIO(b"Lots of dataaaa") http_request = HttpRequest('GET', 'http://127.0.0.1/') http_request.set_streamed_data_body(data) http_retry = RetryPolicy(retry_total = 1) pipeline = Pipeline(MockTransport(), [http_retry]) pipeline.run(http_request)
def _create_config(**kwargs): # type: (Mapping[str, Any]) -> Configuration config = Configuration(**kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) config.retry_policy = RetryPolicy(**kwargs) config.proxy_policy = ProxyPolicy(**kwargs) config.user_agent_policy = UserAgentPolicy(base_user_agent=USER_AGENT, **kwargs) return config
def create_config(**kwargs): # type: (Mapping[str, Any]) -> Configuration config = Configuration(**kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) config.retry_policy = RetryPolicy(retry_on_status_codes=[404, 429] + list(range(500, 600)), **kwargs) return config
def test_retry_types(): history = ["1", "2", "3"] settings = {'history': history, 'backoff': 1, 'max_backoff': 10} retry_policy = RetryPolicy() backoff_time = retry_policy.get_backoff_time(settings) assert backoff_time == 4 retry_policy = RetryPolicy(retry_mode=RetryMode.Fixed) backoff_time = retry_policy.get_backoff_time(settings) assert backoff_time == 1 retry_policy = RetryPolicy(retry_mode=RetryMode.Exponential) backoff_time = retry_policy.get_backoff_time(settings) assert backoff_time == 4
def build_pipeline(transport=None, policies=None, **kwargs): if not policies: config = _get_config(**kwargs) config.retry_policy = RetryPolicy(**kwargs) policies = _get_policies(config, **kwargs) if not transport: transport = RequestsTransport(**kwargs) return Pipeline(transport, policies=policies)
def test_retry_without_http_response(): class NaughtyPolicy(HTTPPolicy): def send(*args): raise AzureError('boo') policies = [RetryPolicy(), NaughtyPolicy()] pipeline = Pipeline(policies=policies, transport=None) with pytest.raises(AzureError): pipeline.run(HttpRequest('GET', url='https://foo.bar'))
def _create_configuration(self, **kwargs): # pylint: disable=no-self-use config = Configuration(**kwargs) config.user_agent_policy = kwargs.get("user_agent_policy") or UserAgentPolicy(**kwargs) config.headers_policy = kwargs.get("headers_policy") or HeadersPolicy(**kwargs) config.proxy_policy = kwargs.get("proxy_policy") or ProxyPolicy(**kwargs) config.logging_policy = kwargs.get("logging_policy") or NetworkTraceLoggingPolicy(**kwargs) config.retry_policy = kwargs.get("retry_policy") or RetryPolicy(**kwargs) config.redirect_policy = kwargs.get("redirect_policy") or RedirectPolicy(**kwargs) return config
def create_config(**kwargs): # type: (Dict[str, str]) -> Configuration timeout = kwargs.pop("connection_timeout", 2) config = Configuration(connection_timeout=timeout, **kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) retries = kwargs.pop("retry_total", 5) config.retry_policy = RetryPolicy(retry_total=retries, retry_on_status_codes=[404, 429] + list(range(500, 600)), **kwargs) return config
def create_config(credential, scopes, **kwargs): # Here the SDK developer would define the default # config to interact with the service config = Configuration(**kwargs) config.headers_policy = HeadersPolicy({"CustomHeader": "Value"}, **kwargs) config.authentication_policy = BearerTokenCredentialPolicy(credential, scopes, **kwargs) config.user_agent_policy = UserAgentPolicy("ServiceUserAgentValue", **kwargs) config.retry_policy = RetryPolicy(**kwargs) config.redirect_policy = RedirectPolicy(**kwargs) config.logging_policy = NetworkTraceLoggingPolicy(**kwargs) config.proxy_policy = ProxyPolicy(**kwargs)
def client(cookie_policy): """Create a AutoRestHttpInfrastructureTestService client with test server credentials.""" policies = [ HeadersPolicy(), ContentDecodePolicy(), RedirectPolicy(), RetryPolicy(), cookie_policy ] with AutoRestHttpInfrastructureTestService( base_url="http://localhost:3000", policies=policies) as client: yield client
def create_config(**kwargs): # type: (Any) -> Configuration config = Configuration(**kwargs) config.user_agent_policy = UserAgentPolicy("KeyClient", **kwargs) config.headers_policy = None config.retry_policy = RetryPolicy(**kwargs) config.redirect_policy = RedirectPolicy(**kwargs) # TODO: these are requests-specific config.cert = config.timeout = None config.verify = True return config
def test_example_no_retries(): url = "https://bing.com" retry_policy = RetryPolicy.no_retries() client = PipelineClient(base_url=url, policies=[retry_policy]) request = client.get(url) pipeline_response = client._pipeline.run(request) response = pipeline_response.http_response # bing returns 301 if not retried assert response.status_code == 301
def __init__(self, **options): self._telemetry_processors = [] self.options = ExporterOptions(**options) retry_policy = RetryPolicy(timeout=self.options.timeout) proxy_policy = ProxyPolicy(proxies=self.options.proxies) self.client = AzureMonitorClient( self.options.endpoint, proxy_policy=proxy_policy, retry_policy=retry_policy) self.storage = LocalFileStorage( path=self.options.storage_path, max_size=self.options.storage_max_size, maintenance_period=self.options.storage_maintenance_period, retention_period=self.options.storage_retention_period, )
def _create_pipeline(**kwargs): """Creates and returns a PipelineClient configured for the provided base_url and kwargs""" transport = kwargs.get("transport", RequestsTransport(**kwargs)) policies = [ kwargs.get("user_agent_policy", UserAgentPolicy(_constants.USER_AGENT, **kwargs)), kwargs.get("headers_policy", HeadersPolicy(**kwargs)), kwargs.get("authentication_policy"), kwargs.get("retry_policy", RetryPolicy(**kwargs)), kwargs.get("redirect_policy", RedirectPolicy(**kwargs)), kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)), kwargs.get("proxy_policy", ProxyPolicy(**kwargs)), ] return Pipeline(policies=policies, transport=transport)
def __init__(self, credential, **kwargs): # pylint:disable=super-init-not-called self._set_universal(**kwargs) # sync-specific azure pipeline policies if isinstance(credential, str): self.headers_policy.add_header("Ocp-Apim-Subscription-Key", credential) else: scopes = kwargs.pop("scopes", []) self.authentication_policy = BearerTokenCredentialPolicy( credential, *scopes, **kwargs) self.retry_policy = kwargs.get("retry_policy", RetryPolicy(**kwargs)) self.redirect_policy = kwargs.get("redirect_policy", RedirectPolicy(**kwargs)) self.transport = kwargs.get("transport", RequestsTransport())
def test_retry_seekable_file(http_request, http_response): class MockTransport(HttpTransport): def __init__(self): self._first = True def __exit__(self, exc_type, exc_val, exc_tb): pass def close(self): pass def open(self): pass def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse if self._first: self._first = False for value in request.files.values(): name, body = value[0], value[1] if name and body and hasattr(body, 'read'): body.seek(0, 2) raise AzureError('fail on first') for value in request.files.values(): name, body = value[0], value[1] if name and body and hasattr(body, 'read'): position = body.tell() assert not position response = create_http_response(http_response, request, None) response.status_code = 400 return response file = tempfile.NamedTemporaryFile(delete=False) file.write(b'Lots of dataaaa') file.close() http_request = http_request('GET', 'http://localhost/') headers = {'Content-Type': "multipart/form-data"} http_request.headers = headers with open(file.name, 'rb') as f: form_data_content = { 'fileContent': f, 'fileName': f.name, } http_request.set_formdata_body(form_data_content) http_retry = RetryPolicy(retry_total=1) pipeline = Pipeline(MockTransport(), [http_retry]) pipeline.run(http_request) os.unlink(f.name)
def test_example_requests(): request = HttpRequest("GET", "https://bing.com") policies = [ UserAgentPolicy("myuseragent"), RedirectPolicy(), RetryPolicy() ] # [START requests] from azure.core.pipeline.transport import RequestsTransport with Pipeline(transport=RequestsTransport(), policies=policies) as pipeline: response = pipeline.run(request) # [END requests] assert pipeline._transport.session is None assert response.http_response.status_code == 200
def test_does_not_sleep_after_timeout(transport_error, expected_timeout_error): # With default settings policy will sleep twice before exhausting its retries: 1.6s, 3.2s. # It should not sleep the second time when given timeout=1 timeout = 1 transport = Mock( spec=HttpTransport, send=Mock(side_effect=transport_error("oops")), sleep=Mock(wraps=time.sleep), ) pipeline = Pipeline(transport, [RetryPolicy(timeout=timeout)]) with pytest.raises(expected_timeout_error): pipeline.run(HttpRequest("GET", "http://127.0.0.1/")) assert transport.sleep.call_count == 1
def test_retry_timeout(): timeout = 1 def send(request, **kwargs): assert kwargs["connection_timeout"] <= timeout, "policy should set connection_timeout not to exceed timeout" raise ServiceResponseError("oops") transport = Mock( spec=HttpTransport, send=Mock(wraps=send), connection_config=ConnectionConfiguration(connection_timeout=timeout * 2), sleep=time.sleep, ) pipeline = Pipeline(transport, [RetryPolicy(timeout=timeout)]) with pytest.raises(ServiceResponseTimeoutError): response = pipeline.run(HttpRequest("GET", "http://127.0.0.1/"))
def test_retry_seekable_file(): class MockTransport(HttpTransport): def __init__(self): self._first = True def __exit__(self, exc_type, exc_val, exc_tb): pass def close(self): pass def open(self): pass def send(self, request, **kwargs): # type: (PipelineRequest, Any) -> PipelineResponse if self._first: self._first = False for value in request.files.values(): name, body = value[0], value[1] if name and body and hasattr(body, 'read'): body.seek(0, 2) raise AzureError('fail on first') for value in request.files.values(): name, body = value[0], value[1] if name and body and hasattr(body, 'read'): position = body.tell() assert not position return HttpResponse(request, None) file_name = 'test_retry_seekable_file' with open(file_name, "w+") as f: f.write('Lots of dataaaa') http_request = HttpRequest('GET', 'http://127.0.0.1/') headers = {'Content-Type': "multipart/form-data"} http_request.headers = headers form_data_content = { 'fileContent': open(file_name, 'rb'), 'fileName': file_name, } http_request.set_formdata_body(form_data_content) http_retry = RetryPolicy(retry_total=1) pipeline = Pipeline(MockTransport(), [http_retry]) pipeline.run(http_request)
def test_timeout_defaults(): """When "timeout" is not set, the policy should not override the transport's timeout configuration""" def send(request, **kwargs): for arg in ("connection_timeout", "read_timeout"): assert arg not in kwargs, "policy should defer to transport configuration when not given a timeout" response = HttpResponse(request, None) response.status_code = 200 return response transport = Mock( spec_set=HttpTransport, send=Mock(wraps=send), sleep=Mock(side_effect=Exception("policy should not sleep: its first send succeeded")), ) pipeline = Pipeline(transport, [RetryPolicy()]) pipeline.run(HttpRequest("GET", "http://127.0.0.1/")) assert transport.send.call_count == 1, "policy should not retry: its first send succeeded"
def _policies(credential, **kwargs): # type: (Union[AzureKeyCredential, EventGridSharedAccessSignatureCredential], Any) -> List[Any] auth_policy = _get_authentication_policy(credential) sdk_moniker = 'eventgrid/{}'.format(VERSION) policies = [ RequestIdPolicy(**kwargs), HeadersPolicy(**kwargs), UserAgentPolicy(sdk_moniker=sdk_moniker, **kwargs), ProxyPolicy(**kwargs), ContentDecodePolicy(**kwargs), RedirectPolicy(**kwargs), RetryPolicy(**kwargs), auth_policy, CustomHookPolicy(**kwargs), NetworkTraceLoggingPolicy(**kwargs), DistributedTracingPolicy(**kwargs), CloudEventDistributedTracingPolicy(), HttpLoggingPolicy(**kwargs) ] return policies