def test_error_writing_to_blobdb(self):
error = ConnectionClosedError(endpoint_url='url')
with self.assertRaises(ConnectionClosedError), patch.object(
get_blob_db(), 'put', side_effect=error):
self._submit('form_with_case.xml')
self.client.exc_info = None # clear error to prevent it from being raised on next request
stubs = UnfinishedSubmissionStub.objects.filter(
domain=self.domain, saved=False, xform_id=FORM_WITH_CASE_ID).all()
self.assertEqual(1, len(stubs))
old_form = XFormInstance.objects.get_form(FORM_WITH_CASE_ID,
self.domain)
self.assertTrue(old_form.is_error)
self.assertTrue(old_form.initial_processing_complete)
expected_problem_message = f'{type(error).__name__}: {error}'
self.assertEqual(old_form.problem, expected_problem_message)
_, resp = self._submit('form_with_case.xml')
self.assertEqual(resp.status_code, 201)
new_form = XFormInstance.objects.get_form(FORM_WITH_CASE_ID,
self.domain)
self.assertTrue(new_form.is_normal)
old_form.refresh_from_db(
) # can't fetch by form_id since form_id changed
self.assertEqual(old_form.orig_id, FORM_WITH_CASE_ID)
self.assertEqual(old_form.problem, expected_problem_message)
async def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
# http request substituted too async one
logger.debug("Sending http request: %s", request)
history_recorder.record(
'HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
http_response = await self._request(
request.method, request.url, request.headers, request.body,
operation_model.has_streaming_output)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
if self._looks_like_dns_error(e):
better_exception = EndpointConnectionError(
endpoint_url=request.url, error=e)
return None, better_exception
else:
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(endpoint_url=request.url,
request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
parser = self._response_parser_factory.create_parser(
operation_model.metadata['protocol'])
parsed_response = parser.parse(response_dict,
operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def test_http_connection_errors_is_retried(self):
# Connection related errors should be retried
self.add_imds_token_response()
self.add_imds_connection_error(ConnectionClosedError(endpoint_url=''))
self.add_get_region_imds_response()
result = InstanceMetadataRegionFetcher(
num_attempts=2).retrieve_region()
expected_result = 'us-mars-1'
self.assertEqual(result, expected_result)
def test_catch_retryable_http_errors(self):
with mock.patch('botocore.httpsession.URLLib3Session.send') as send_mock:
fetcher = InstanceMetadataFetcher()
send_mock.side_effect = ConnectionClosedError(endpoint_url="foo")
creds = fetcher.retrieve_iam_role_credentials()
self.assertEqual(send_mock.call_count, 2)
for call_instance in send_mock.call_args_list:
self.assertTrue(call_instance[0][0].url.startswith(fetcher.get_base_url()))
self.assertEqual(creds, {})
def test_request_retried_for_sigv4(self):
body = six.BytesIO(b"Hello world!")
exception = ConnectionClosedError(endpoint_url='')
self.http_stubber.responses.append(exception)
self.http_stubber.responses.append(None)
with self.http_stubber:
response = self.client.put_object(Bucket=self.bucket_name,
Key='foo.txt', Body=body)
self.assert_status_code(response, 200)
async def _do_get_response(self, request, operation_model):
try:
logger.debug("Sending http request: %s", request)
history_recorder.record('HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % (
service_id, operation_model.name)
responses = await self._event_emitter.emit(event_name,
request=request)
http_response = first_non_none_response(responses)
if http_response is None:
http_response = await self._send(request)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(
endpoint_url=request.url, request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
if asyncio.iscoroutinefunction(parser.parse):
parsed_response = await parser.parse(
response_dict, operation_model.output_shape)
else:
parsed_response = parser.parse(
response_dict, operation_model.output_shape)
if http_response.status_code >= 300:
await self._add_modeled_error_fields(
response_dict, parsed_response,
operation_model, parser,
)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def send(self, request):
try:
proxy_url = self._proxy_config.proxy_url_for(request.url)
manager = self._get_connection_manager(request.url, proxy_url)
conn = manager.connection_from_url(request.url)
self._setup_ssl_cert(conn, request.url, self._verify)
request_target = self._get_request_target(request.url, proxy_url)
urllib_response = conn.urlopen(
method=request.method,
url=request_target,
body=request.body,
headers=request.headers,
retries=Retry(False),
assert_same_host=False,
preload_content=False,
decode_content=False,
chunked=self._chunked(request.headers),
)
http_response = botocore.awsrequest.AWSResponse(
request.url,
urllib_response.status,
urllib_response.headers,
urllib_response,
)
if not request.stream_output:
# Cause the raw stream to be exhausted immediately. We do it
# this way instead of using preload_content because
# preload_content will never buffer chunked responses
http_response.content
return http_response
except URLLib3SSLError as e:
raise SSLError(endpoint_url=request.url, error=e)
except (NewConnectionError, socket.gaierror) as e:
raise EndpointConnectionError(endpoint_url=request.url, error=e)
except ProxyError as e:
raise ProxyConnectionError(proxy_url=proxy_url, error=e)
except URLLib3ConnectTimeoutError as e:
raise ConnectTimeoutError(endpoint_url=request.url, error=e)
except URLLib3ReadTimeoutError as e:
raise ReadTimeoutError(endpoint_url=request.url, error=e)
except ProtocolError as e:
raise ConnectionClosedError(
error=e,
request=request,
endpoint_url=request.url
)
except Exception as e:
message = 'Exception received when sending urllib3 HTTP request'
logger.debug(message, exc_info=True)
raise HTTPClientError(error=e)
def _make_client_call_with_errors(client, operation_name, kwargs):
operation = getattr(client, xform_name(operation_name))
exception = ConnectionClosedError(endpoint_url='https://mock.eror')
with ClientHTTPStubber(client) as http_stubber:
http_stubber.responses.append(exception)
http_stubber.responses.append(None)
try:
response = operation(**kwargs)
except ClientError as e:
assert False, ('Request was not retried properly, '
'received error:\n%s' % pformat(e))
def _make_client_call_with_errors(client, operation_name, kwargs):
operation = getattr(client, xform_name(operation_name))
exception = ConnectionClosedError(endpoint_url='https://mock.eror')
with ClientHTTPStubber(client, strict=False) as http_stubber:
http_stubber.responses.append(exception)
try:
response = operation(**kwargs)
except ClientError as e:
assert False, ('Request was not retried properly, '
'received error:\n%s' % pformat(e))
# Ensure we used the stubber as we're not using it in strict mode
assert len(http_stubber.responses) == 0, 'Stubber was not used!'
def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
# http request substituted too async one
botocore.endpoint.logger.debug("Sending http request: %s", request)
# TODO: handle self.proxies
resp = yield from self._request(request.method, request.url,
request.headers, request.body)
http_response = resp
except aiohttp.errors.BadStatusLine:
better_exception = ConnectionClosedError(endpoint_url=request.url,
request=request)
return None, better_exception
except aiohttp.errors.ClientConnectionError as e:
e.request = request # botocore expects the request property
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
botocore.endpoint.logger.debug(
"ConnectionError received when "
"sending HTTP request.",
exc_info=True)
if self._looks_like_dns_error(e):
better_exception = EndpointConnectionError(
endpoint_url=request.url, error=e)
return None, better_exception
else:
return None, e
except Exception as e:
botocore.endpoint.logger.debug(
"Exception received when sending "
"HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = yield from convert_to_response_dict(
http_response, operation_model)
parser = self._response_parser_factory.create_parser(
operation_model.metadata['protocol'])
parsed_response = parser.parse(response_dict,
operation_model.output_shape)
return (http_response, parsed_response), None
def test_default_aws_retry():
# AssertionError should not retry
e = AssertionError()
assert not default_aws_retry(e)
# NotImplementedError should not retry
e = NotImplementedError()
assert not default_aws_retry(e)
# KeyError should not retry
e = KeyError()
assert not default_aws_retry(e)
# IndexError should not retry
e = IndexError()
assert not default_aws_retry(e)
# EndpointConnectionError should retry
e = EndpointConnectionError(endpoint_url=None, error=None)
assert default_aws_retry(e)
# ConnectionClosedError should retry
e = ConnectionClosedError(endpoint_url=None)
assert default_aws_retry(e)
# Generic BotoCoreError should not retry
e = BotoCoreError()
assert not default_aws_retry(e)
# Generic ClientError should not retry
err = {
"Error": {
"Code": "xxx",
"Message": "xxx"
}
}
e = ClientError(error_response=err, operation_name="xxx")
assert not default_aws_retry(e)
# ClientError with RequestLimitExceeded should retry
err = {
"Error": {
"Code": "RequestLimitExceeded",
"Message": "xxx"
}
}
e = ClientError(error_response=err, operation_name="xxx")
assert default_aws_retry(e)
async def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
logger.debug("Sending http request: %s", request)
headers = dict(self._headers(request.headers))
http_response = await self.http_session.request(
method=request.method,
url=request.url,
data=request.body,
headers=headers,
stream=True,
verify=self.verify)
except ConnectionError as e:
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
if self._looks_like_dns_error(e):
endpoint_url = e.request.url
better_exception = EndpointConnectionError(
endpoint_url=endpoint_url, error=e)
return (None, better_exception)
elif self._looks_like_bad_status_line(e):
better_exception = ConnectionClosedError(
endpoint_url=e.request.url, request=e.request)
return (None, better_exception)
else:
return (None, e)
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return (None, e)
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
parser = self._response_parser_factory.create_parser(
operation_model.metadata['protocol'])
return ((http_response,
parser.parse(response_dict,
operation_model.output_shape)), None)
def test_error_writing_to_blobdb(self):
error = ConnectionClosedError(endpoint_url='url')
with self.assertRaises(ConnectionClosedError), patch.object(
get_blob_db(), 'put', side_effect=error):
self._submit('form_with_case.xml')
stubs = UnfinishedSubmissionStub.objects.filter(
domain=self.domain, saved=False, xform_id=FORM_WITH_CASE_ID).all()
self.assertEqual(1, len(stubs))
form = FormAccessors(self.domain).get_form(FORM_WITH_CASE_ID)
self.assertTrue(form.is_error)
self.assertTrue(form.initial_processing_complete)
expected_problem_message = f'{type(error).__name__}: {error}'
self.assertEqual(form.problem, expected_problem_message)
_, resp = self._submit('form_with_case.xml')
self.assertEqual(resp.status_code, 201)
form = FormAccessors(self.domain).get_form(FORM_WITH_CASE_ID)
self.assertTrue(form.is_normal)
old_form = FormAccessors(self.domain).get_form(form.deprecated_form_id)
self.assertEqual(old_form.orig_id, FORM_WITH_CASE_ID)
self.assertEqual(old_form.problem, expected_problem_message)
def send(self, request):
try:
proxy_url = self._proxy_config.proxy_url_for(request.url)
manager = self._get_connection_manager(request.url, proxy_url)
conn = manager.connection_from_url(request.url)
self._setup_ssl_cert(conn, request.url, self._verify)
if ensure_boolean(
os.environ.get('BOTO_EXPERIMENTAL__ADD_PROXY_HOST_HEADER',
'')):
# This is currently an "experimental" feature which provides
# no guarantees of backwards compatibility. It may be subject
# to change or removal in any patch version. Anyone opting in
# to this feature should strictly pin botocore.
host = urlparse(request.url).hostname
conn.proxy_headers['host'] = host
request_target = self._get_request_target(request.url, proxy_url)
urllib_response = conn.urlopen(
method=request.method,
url=request_target,
body=request.body,
headers=request.headers,
retries=Retry(False),
assert_same_host=False,
preload_content=False,
decode_content=False,
chunked=self._chunked(request.headers),
)
http_response = botocore.awsrequest.AWSResponse(
request.url,
urllib_response.status,
urllib_response.headers,
urllib_response,
)
if not request.stream_output:
# Cause the raw stream to be exhausted immediately. We do it
# this way instead of using preload_content because
# preload_content will never buffer chunked responses
http_response.content
return http_response
except URLLib3SSLError as e:
raise SSLError(endpoint_url=request.url, error=e)
except (NewConnectionError, socket.gaierror) as e:
raise EndpointConnectionError(endpoint_url=request.url, error=e)
except ProxyError as e:
raise ProxyConnectionError(proxy_url=proxy_url, error=e)
except URLLib3ConnectTimeoutError as e:
raise ConnectTimeoutError(endpoint_url=request.url, error=e)
except URLLib3ReadTimeoutError as e:
raise ReadTimeoutError(endpoint_url=request.url, error=e)
except ProtocolError as e:
raise ConnectionClosedError(error=e,
request=request,
endpoint_url=request.url)
except Exception as e:
message = 'Exception received when sending urllib3 HTTP request'
logger.debug(message, exc_info=True)
raise HTTPClientError(error=e)
async def _do_get_response(self, request, operation_model):
try:
# http request substituted too async one
logger.debug("Sending http request: %s", request)
history_recorder.record('HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % \
(service_id, operation_model.name)
responses = self._event_emitter.emit(event_name, request=request)
http_response = first_non_none_response(responses)
if http_response is None:
streaming = any([
operation_model.has_streaming_output,
operation_model.has_event_stream_output
])
http_response = await self._request(
request.method, request.url, request.headers, request.body,
verify=self.verify_ssl,
stream=streaming)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(
endpoint_url=request.url, request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
parsed_response = parser.parse(
response_dict, operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def mock_endpoint_send(self, *args, **kwargs):
if not getattr(self, '_integ_test_error_raised', False):
self._integ_test_error_raised = True
raise ConnectionClosedError(endpoint_url='')
else:
return original_send(self, *args, **kwargs)
def mock_endpoint_send(self, *args, **kwargs):
if not state.error_raised:
state.error_raised = True
raise ConnectionClosedError(endpoint_url='')
else:
return original_send(self, *args, **kwargs)
def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
logger.debug("Sending http request: %s", request)
history_recorder.record(
'HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
http_response = self.http_session.send(
request,
verify=self.verify,
stream=operation_model.has_streaming_output,
proxies=self.proxies,
timeout=self.timeout)
except ConnectionError as e:
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
if self._looks_like_dns_error(e):
endpoint_url = e.request.url
better_exception = EndpointConnectionError(
endpoint_url=endpoint_url, error=e)
return (None, better_exception)
elif self._looks_like_bad_status_line(e):
better_exception = ConnectionClosedError(
endpoint_url=e.request.url, request=e.request)
return (None, better_exception)
else:
return (None, e)
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return (None, e)
# This returns the http_response and the parsed_data.
response_dict = convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
if http_response.status_code == 200 and 'callback' in operation_model.metadata:
parser = self._response_parser_factory.create_parser("rest-json")
operation_model.metadata['protocol'] = "rest-json"
else:
parser = self._response_parser_factory.create_parser(
operation_model.metadata['protocol'])
parsed_response = parser.parse(response_dict,
operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
