def create_endpoint(self, service_model, region_name, endpoint_url,
verify=None, response_parser_factory=None,
timeout=DEFAULT_TIMEOUT,
max_pool_connections=MAX_POOL_CONNECTIONS,
http_session_cls=AIOHTTPSession,
proxies=None,
socket_options=None,
client_cert=None,
proxies_config=None,
connector_args=None):
if not is_valid_endpoint_url(endpoint_url):
raise ValueError("Invalid endpoint: %s" % endpoint_url)
if proxies is None:
proxies = self._get_proxies(endpoint_url)
endpoint_prefix = service_model.endpoint_prefix
logger.debug('Setting %s timeout as %s', endpoint_prefix, timeout)
http_session = http_session_cls(
timeout=timeout,
proxies=proxies,
verify=self._get_verify_value(verify),
max_pool_connections=max_pool_connections,
socket_options=socket_options,
client_cert=client_cert,
proxies_config=proxies_config,
connector_args=connector_args
)
return AioEndpoint(
endpoint_url,
endpoint_prefix=endpoint_prefix,
event_emitter=self._event_emitter,
response_parser_factory=response_parser_factory,
http_session=http_session)
async def _needs_retry(self,
attempts,
operation_model,
request_dict,
response=None,
caught_exception=None):
event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix,
operation_model.name)
responses = self._event_emitter.emit(event_name,
response=response,
endpoint=self,
operation=operation_model,
attempts=attempts,
caught_exception=caught_exception,
request_dict=request_dict)
handler_response = first_non_none_response(responses)
if handler_response is None:
return False
else:
# Request needs to be retried, and we need to sleep
# for the specified number of times.
logger.debug(
"Response received to retry, sleeping for "
"%s seconds", handler_response)
await asyncio.sleep(handler_response, loop=self._loop)
return True
async def _do_get_response(self, request, operation_model):
try:
logger.debug("Sending http request: %s", request)
history_recorder.record('HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % (
service_id, operation_model.name)
responses = await self._event_emitter.emit(event_name,
request=request)
http_response = first_non_none_response(responses)
if http_response is None:
http_response = await self._send(request)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(
endpoint_url=request.url, request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
if asyncio.iscoroutinefunction(parser.parse):
parsed_response = await parser.parse(
response_dict, operation_model.output_shape)
else:
parsed_response = parser.parse(
response_dict, operation_model.output_shape)
if http_response.status_code >= 300:
await self._add_modeled_error_fields(
response_dict, parsed_response,
operation_model, parser,
)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
async def _needs_retry(self, attempts, operation_model, request_dict,
response=None, caught_exception=None):
event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix,
operation_model.name)
responses = self._event_emitter.emit(
event_name, response=response, endpoint=self,
operation=operation_model, attempts=attempts,
caught_exception=caught_exception, request_dict=request_dict)
handler_response = first_non_none_response(responses)
if handler_response is None:
return False
else:
# Request needs to be retried, and we need to sleep
# for the specified number of times.
logger.debug("Response received to retry, sleeping for "
"%s seconds", handler_response)
await asyncio.sleep(handler_response, loop=self._loop)
return True
async def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
# http request substituted too async one
logger.debug("Sending http request: %s", request)
history_recorder.record(
'HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
http_response = await self._request(
request.method, request.url, request.headers, request.body,
operation_model.has_streaming_output)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
if self._looks_like_dns_error(e):
better_exception = EndpointConnectionError(
endpoint_url=request.url, error=e)
return None, better_exception
else:
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(endpoint_url=request.url,
request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
parser = self._response_parser_factory.create_parser(
operation_model.metadata['protocol'])
parsed_response = parser.parse(response_dict,
operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def create_endpoint(self,
service_model,
region_name,
endpoint_url,
verify=None,
response_parser_factory=None,
timeout=DEFAULT_TIMEOUT,
max_pool_connections=MAX_POOL_CONNECTIONS,
http_session_cls=aiohttp.ClientSession,
proxies=None,
socket_options=None,
client_cert=None,
proxies_config=None,
connector_args=None):
if not is_valid_endpoint_url(endpoint_url):
raise ValueError("Invalid endpoint: %s" % endpoint_url)
if proxies is None:
proxies = self._get_proxies(endpoint_url)
endpoint_prefix = service_model.endpoint_prefix
logger.debug('Setting %s timeout as %s', endpoint_prefix, timeout)
if isinstance(timeout, (list, tuple)):
conn_timeout, read_timeout = timeout
else:
conn_timeout = read_timeout = timeout
if connector_args is None:
# AWS has a 20 second idle timeout:
# https://forums.aws.amazon.com/message.jspa?messageID=215367
# aiohttp default timeout is 30s so set something reasonable here
connector_args = dict(keepalive_timeout=12)
timeout = aiohttp.ClientTimeout(sock_connect=conn_timeout,
sock_read=read_timeout)
verify = self._get_verify_value(verify)
ssl_context = None
if client_cert:
if isinstance(client_cert, str):
key_file = None
cert_file = client_cert
elif isinstance(client_cert, tuple):
cert_file, key_file = client_cert
else:
raise TypeError("client_cert must be str or tuple, not %s" %
client_cert.__class__.__name__)
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_context.load_cert_chain(cert_file, key_file)
elif isinstance(verify, (str, pathlib.Path)):
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH,
cafile=str(verify))
# TODO: add support for proxies_config
connector = aiohttp.TCPConnector(limit=max_pool_connections,
verify_ssl=bool(verify),
ssl=ssl_context,
**connector_args)
aio_session = http_session_cls(connector=connector,
timeout=timeout,
skip_auto_headers={'CONTENT-TYPE'},
response_class=ClientResponseProxy,
auto_decompress=False)
return AioEndpoint(endpoint_url,
endpoint_prefix=endpoint_prefix,
event_emitter=self._event_emitter,
response_parser_factory=response_parser_factory,
http_session=aio_session,
proxies=proxies)
async def _do_get_response(self, request, operation_model):
try:
# http request substituted too async one
logger.debug("Sending http request: %s", request)
history_recorder.record('HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % \
(service_id, operation_model.name)
responses = self._event_emitter.emit(event_name, request=request)
http_response = first_non_none_response(responses)
if http_response is None:
streaming = any([
operation_model.has_streaming_output,
operation_model.has_event_stream_output
])
http_response = await self._request(
request.method, request.url, request.headers, request.body,
verify=self.verify_ssl,
stream=streaming)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(
endpoint_url=request.url, request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
parsed_response = parser.parse(
response_dict, operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
