def test_error_writing_to_blobdb(self): error = ConnectionClosedError(endpoint_url='url') with self.assertRaises(ConnectionClosedError), patch.object( get_blob_db(), 'put', side_effect=error): self._submit('form_with_case.xml') self.client.exc_info = None # clear error to prevent it from being raised on next request stubs = UnfinishedSubmissionStub.objects.filter( domain=self.domain, saved=False, xform_id=FORM_WITH_CASE_ID).all() self.assertEqual(1, len(stubs)) old_form = XFormInstance.objects.get_form(FORM_WITH_CASE_ID, self.domain) self.assertTrue(old_form.is_error) self.assertTrue(old_form.initial_processing_complete) expected_problem_message = f'{type(error).__name__}: {error}' self.assertEqual(old_form.problem, expected_problem_message) _, resp = self._submit('form_with_case.xml') self.assertEqual(resp.status_code, 201) new_form = XFormInstance.objects.get_form(FORM_WITH_CASE_ID, self.domain) self.assertTrue(new_form.is_normal) old_form.refresh_from_db( ) # can't fetch by form_id since form_id changed self.assertEqual(old_form.orig_id, FORM_WITH_CASE_ID) self.assertEqual(old_form.problem, expected_problem_message)
async def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: # http request substituted too async one logger.debug("Sending http request: %s", request) history_recorder.record( 'HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) http_response = await self._request( request.method, request.url, request.headers, request.body, operation_model.has_streaming_output) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) if self._looks_like_dns_error(e): better_exception = EndpointConnectionError( endpoint_url=request.url, error=e) return None, better_exception else: return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError(endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) parser = self._response_parser_factory.create_parser( operation_model.metadata['protocol']) parsed_response = parser.parse(response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def test_http_connection_errors_is_retried(self): # Connection related errors should be retried self.add_imds_token_response() self.add_imds_connection_error(ConnectionClosedError(endpoint_url='')) self.add_get_region_imds_response() result = InstanceMetadataRegionFetcher( num_attempts=2).retrieve_region() expected_result = 'us-mars-1' self.assertEqual(result, expected_result)
def test_catch_retryable_http_errors(self): with mock.patch('botocore.httpsession.URLLib3Session.send') as send_mock: fetcher = InstanceMetadataFetcher() send_mock.side_effect = ConnectionClosedError(endpoint_url="foo") creds = fetcher.retrieve_iam_role_credentials() self.assertEqual(send_mock.call_count, 2) for call_instance in send_mock.call_args_list: self.assertTrue(call_instance[0][0].url.startswith(fetcher.get_base_url())) self.assertEqual(creds, {})
def test_request_retried_for_sigv4(self): body = six.BytesIO(b"Hello world!") exception = ConnectionClosedError(endpoint_url='') self.http_stubber.responses.append(exception) self.http_stubber.responses.append(None) with self.http_stubber: response = self.client.put_object(Bucket=self.bucket_name, Key='foo.txt', Body=body) self.assert_status_code(response, 200)
async def _do_get_response(self, request, operation_model): try: logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % ( service_id, operation_model.name) responses = await self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: http_response = await self._send(request) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError( endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) if asyncio.iscoroutinefunction(parser.parse): parsed_response = await parser.parse( response_dict, operation_model.output_shape) else: parsed_response = parser.parse( response_dict, operation_model.output_shape) if http_response.status_code >= 300: await self._add_modeled_error_fields( response_dict, parsed_response, operation_model, parser, ) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def send(self, request): try: proxy_url = self._proxy_config.proxy_url_for(request.url) manager = self._get_connection_manager(request.url, proxy_url) conn = manager.connection_from_url(request.url) self._setup_ssl_cert(conn, request.url, self._verify) request_target = self._get_request_target(request.url, proxy_url) urllib_response = conn.urlopen( method=request.method, url=request_target, body=request.body, headers=request.headers, retries=Retry(False), assert_same_host=False, preload_content=False, decode_content=False, chunked=self._chunked(request.headers), ) http_response = botocore.awsrequest.AWSResponse( request.url, urllib_response.status, urllib_response.headers, urllib_response, ) if not request.stream_output: # Cause the raw stream to be exhausted immediately. We do it # this way instead of using preload_content because # preload_content will never buffer chunked responses http_response.content return http_response except URLLib3SSLError as e: raise SSLError(endpoint_url=request.url, error=e) except (NewConnectionError, socket.gaierror) as e: raise EndpointConnectionError(endpoint_url=request.url, error=e) except ProxyError as e: raise ProxyConnectionError(proxy_url=proxy_url, error=e) except URLLib3ConnectTimeoutError as e: raise ConnectTimeoutError(endpoint_url=request.url, error=e) except URLLib3ReadTimeoutError as e: raise ReadTimeoutError(endpoint_url=request.url, error=e) except ProtocolError as e: raise ConnectionClosedError( error=e, request=request, endpoint_url=request.url ) except Exception as e: message = 'Exception received when sending urllib3 HTTP request' logger.debug(message, exc_info=True) raise HTTPClientError(error=e)
def _make_client_call_with_errors(client, operation_name, kwargs): operation = getattr(client, xform_name(operation_name)) exception = ConnectionClosedError(endpoint_url='https://mock.eror') with ClientHTTPStubber(client) as http_stubber: http_stubber.responses.append(exception) http_stubber.responses.append(None) try: response = operation(**kwargs) except ClientError as e: assert False, ('Request was not retried properly, ' 'received error:\n%s' % pformat(e))
def _make_client_call_with_errors(client, operation_name, kwargs): operation = getattr(client, xform_name(operation_name)) exception = ConnectionClosedError(endpoint_url='https://mock.eror') with ClientHTTPStubber(client, strict=False) as http_stubber: http_stubber.responses.append(exception) try: response = operation(**kwargs) except ClientError as e: assert False, ('Request was not retried properly, ' 'received error:\n%s' % pformat(e)) # Ensure we used the stubber as we're not using it in strict mode assert len(http_stubber.responses) == 0, 'Stubber was not used!'
def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: # http request substituted too async one botocore.endpoint.logger.debug("Sending http request: %s", request) # TODO: handle self.proxies resp = yield from self._request(request.method, request.url, request.headers, request.body) http_response = resp except aiohttp.errors.BadStatusLine: better_exception = ConnectionClosedError(endpoint_url=request.url, request=request) return None, better_exception except aiohttp.errors.ClientConnectionError as e: e.request = request # botocore expects the request property # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. botocore.endpoint.logger.debug( "ConnectionError received when " "sending HTTP request.", exc_info=True) if self._looks_like_dns_error(e): better_exception = EndpointConnectionError( endpoint_url=request.url, error=e) return None, better_exception else: return None, e except Exception as e: botocore.endpoint.logger.debug( "Exception received when sending " "HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = yield from convert_to_response_dict( http_response, operation_model) parser = self._response_parser_factory.create_parser( operation_model.metadata['protocol']) parsed_response = parser.parse(response_dict, operation_model.output_shape) return (http_response, parsed_response), None
def test_default_aws_retry(): # AssertionError should not retry e = AssertionError() assert not default_aws_retry(e) # NotImplementedError should not retry e = NotImplementedError() assert not default_aws_retry(e) # KeyError should not retry e = KeyError() assert not default_aws_retry(e) # IndexError should not retry e = IndexError() assert not default_aws_retry(e) # EndpointConnectionError should retry e = EndpointConnectionError(endpoint_url=None, error=None) assert default_aws_retry(e) # ConnectionClosedError should retry e = ConnectionClosedError(endpoint_url=None) assert default_aws_retry(e) # Generic BotoCoreError should not retry e = BotoCoreError() assert not default_aws_retry(e) # Generic ClientError should not retry err = { "Error": { "Code": "xxx", "Message": "xxx" } } e = ClientError(error_response=err, operation_name="xxx") assert not default_aws_retry(e) # ClientError with RequestLimitExceeded should retry err = { "Error": { "Code": "RequestLimitExceeded", "Message": "xxx" } } e = ClientError(error_response=err, operation_name="xxx") assert default_aws_retry(e)
async def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: logger.debug("Sending http request: %s", request) headers = dict(self._headers(request.headers)) http_response = await self.http_session.request( method=request.method, url=request.url, data=request.body, headers=headers, stream=True, verify=self.verify) except ConnectionError as e: # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) if self._looks_like_dns_error(e): endpoint_url = e.request.url better_exception = EndpointConnectionError( endpoint_url=endpoint_url, error=e) return (None, better_exception) elif self._looks_like_bad_status_line(e): better_exception = ConnectionClosedError( endpoint_url=e.request.url, request=e.request) return (None, better_exception) else: return (None, e) except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return (None, e) # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) parser = self._response_parser_factory.create_parser( operation_model.metadata['protocol']) return ((http_response, parser.parse(response_dict, operation_model.output_shape)), None)
def test_error_writing_to_blobdb(self): error = ConnectionClosedError(endpoint_url='url') with self.assertRaises(ConnectionClosedError), patch.object( get_blob_db(), 'put', side_effect=error): self._submit('form_with_case.xml') stubs = UnfinishedSubmissionStub.objects.filter( domain=self.domain, saved=False, xform_id=FORM_WITH_CASE_ID).all() self.assertEqual(1, len(stubs)) form = FormAccessors(self.domain).get_form(FORM_WITH_CASE_ID) self.assertTrue(form.is_error) self.assertTrue(form.initial_processing_complete) expected_problem_message = f'{type(error).__name__}: {error}' self.assertEqual(form.problem, expected_problem_message) _, resp = self._submit('form_with_case.xml') self.assertEqual(resp.status_code, 201) form = FormAccessors(self.domain).get_form(FORM_WITH_CASE_ID) self.assertTrue(form.is_normal) old_form = FormAccessors(self.domain).get_form(form.deprecated_form_id) self.assertEqual(old_form.orig_id, FORM_WITH_CASE_ID) self.assertEqual(old_form.problem, expected_problem_message)
def send(self, request): try: proxy_url = self._proxy_config.proxy_url_for(request.url) manager = self._get_connection_manager(request.url, proxy_url) conn = manager.connection_from_url(request.url) self._setup_ssl_cert(conn, request.url, self._verify) if ensure_boolean( os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER', '')): # This is currently an "experimental" feature which provides # no guarantees of backwards compatibility. It may be subject # to change or removal in any patch version. Anyone opting in # to this feature should strictly pin botocore. host = urlparse(request.url).hostname conn.proxy_headers['host'] = host request_target = self._get_request_target(request.url, proxy_url) urllib_response = conn.urlopen( method=request.method, url=request_target, body=request.body, headers=request.headers, retries=Retry(False), assert_same_host=False, preload_content=False, decode_content=False, chunked=self._chunked(request.headers), ) http_response = botocore.awsrequest.AWSResponse( request.url, urllib_response.status, urllib_response.headers, urllib_response, ) if not request.stream_output: # Cause the raw stream to be exhausted immediately. We do it # this way instead of using preload_content because # preload_content will never buffer chunked responses http_response.content return http_response except URLLib3SSLError as e: raise SSLError(endpoint_url=request.url, error=e) except (NewConnectionError, socket.gaierror) as e: raise EndpointConnectionError(endpoint_url=request.url, error=e) except ProxyError as e: raise ProxyConnectionError(proxy_url=proxy_url, error=e) except URLLib3ConnectTimeoutError as e: raise ConnectTimeoutError(endpoint_url=request.url, error=e) except URLLib3ReadTimeoutError as e: raise ReadTimeoutError(endpoint_url=request.url, error=e) except ProtocolError as e: raise ConnectionClosedError(error=e, request=request, endpoint_url=request.url) except Exception as e: message = 'Exception received when sending urllib3 HTTP request' logger.debug(message, exc_info=True) raise HTTPClientError(error=e)
async def _do_get_response(self, request, operation_model): try: # http request substituted too async one logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % \ (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: streaming = any([ operation_model.has_streaming_output, operation_model.has_event_stream_output ]) http_response = await self._request( request.method, request.url, request.headers, request.body, verify=self.verify_ssl, stream=streaming) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError( endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) parsed_response = parser.parse( response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def mock_endpoint_send(self, *args, **kwargs): if not getattr(self, '_integ_test_error_raised', False): self._integ_test_error_raised = True raise ConnectionClosedError(endpoint_url='') else: return original_send(self, *args, **kwargs)
def mock_endpoint_send(self, *args, **kwargs): if not state.error_raised: state.error_raised = True raise ConnectionClosedError(endpoint_url='') else: return original_send(self, *args, **kwargs)
def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: logger.debug("Sending http request: %s", request) history_recorder.record( 'HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) http_response = self.http_session.send( request, verify=self.verify, stream=operation_model.has_streaming_output, proxies=self.proxies, timeout=self.timeout) except ConnectionError as e: # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) if self._looks_like_dns_error(e): endpoint_url = e.request.url better_exception = EndpointConnectionError( endpoint_url=endpoint_url, error=e) return (None, better_exception) elif self._looks_like_bad_status_line(e): better_exception = ConnectionClosedError( endpoint_url=e.request.url, request=e.request) return (None, better_exception) else: return (None, e) except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return (None, e) # This returns the http_response and the parsed_data. response_dict = convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) if http_response.status_code == 200 and 'callback' in operation_model.metadata: parser = self._response_parser_factory.create_parser("rest-json") operation_model.metadata['protocol'] = "rest-json" else: parser = self._response_parser_factory.create_parser( operation_model.metadata['protocol']) parsed_response = parser.parse(response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None