def create_endpoint(self, service_model, region_name, endpoint_url, verify=None, response_parser_factory=None, timeout=DEFAULT_TIMEOUT, max_pool_connections=MAX_POOL_CONNECTIONS, http_session_cls=AIOHTTPSession, proxies=None, socket_options=None, client_cert=None, proxies_config=None, connector_args=None): if not is_valid_endpoint_url(endpoint_url): raise ValueError("Invalid endpoint: %s" % endpoint_url) if proxies is None: proxies = self._get_proxies(endpoint_url) endpoint_prefix = service_model.endpoint_prefix logger.debug('Setting %s timeout as %s', endpoint_prefix, timeout) http_session = http_session_cls( timeout=timeout, proxies=proxies, verify=self._get_verify_value(verify), max_pool_connections=max_pool_connections, socket_options=socket_options, client_cert=client_cert, proxies_config=proxies_config, connector_args=connector_args ) return AioEndpoint( endpoint_url, endpoint_prefix=endpoint_prefix, event_emitter=self._event_emitter, response_parser_factory=response_parser_factory, http_session=http_session)
async def _needs_retry(self, attempts, operation_model, request_dict, response=None, caught_exception=None): event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix, operation_model.name) responses = self._event_emitter.emit(event_name, response=response, endpoint=self, operation=operation_model, attempts=attempts, caught_exception=caught_exception, request_dict=request_dict) handler_response = first_non_none_response(responses) if handler_response is None: return False else: # Request needs to be retried, and we need to sleep # for the specified number of times. logger.debug( "Response received to retry, sleeping for " "%s seconds", handler_response) await asyncio.sleep(handler_response, loop=self._loop) return True
async def _do_get_response(self, request, operation_model): try: logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % ( service_id, operation_model.name) responses = await self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: http_response = await self._send(request) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError( endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) if asyncio.iscoroutinefunction(parser.parse): parsed_response = await parser.parse( response_dict, operation_model.output_shape) else: parsed_response = parser.parse( response_dict, operation_model.output_shape) if http_response.status_code >= 300: await self._add_modeled_error_fields( response_dict, parsed_response, operation_model, parser, ) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
async def _needs_retry(self, attempts, operation_model, request_dict, response=None, caught_exception=None): event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix, operation_model.name) responses = self._event_emitter.emit( event_name, response=response, endpoint=self, operation=operation_model, attempts=attempts, caught_exception=caught_exception, request_dict=request_dict) handler_response = first_non_none_response(responses) if handler_response is None: return False else: # Request needs to be retried, and we need to sleep # for the specified number of times. logger.debug("Response received to retry, sleeping for " "%s seconds", handler_response) await asyncio.sleep(handler_response, loop=self._loop) return True
async def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: # http request substituted too async one logger.debug("Sending http request: %s", request) history_recorder.record( 'HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) http_response = await self._request( request.method, request.url, request.headers, request.body, operation_model.has_streaming_output) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) if self._looks_like_dns_error(e): better_exception = EndpointConnectionError( endpoint_url=request.url, error=e) return None, better_exception else: return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError(endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) parser = self._response_parser_factory.create_parser( operation_model.metadata['protocol']) parsed_response = parser.parse(response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def create_endpoint(self, service_model, region_name, endpoint_url, verify=None, response_parser_factory=None, timeout=DEFAULT_TIMEOUT, max_pool_connections=MAX_POOL_CONNECTIONS, http_session_cls=aiohttp.ClientSession, proxies=None, socket_options=None, client_cert=None, proxies_config=None, connector_args=None): if not is_valid_endpoint_url(endpoint_url): raise ValueError("Invalid endpoint: %s" % endpoint_url) if proxies is None: proxies = self._get_proxies(endpoint_url) endpoint_prefix = service_model.endpoint_prefix logger.debug('Setting %s timeout as %s', endpoint_prefix, timeout) if isinstance(timeout, (list, tuple)): conn_timeout, read_timeout = timeout else: conn_timeout = read_timeout = timeout if connector_args is None: # AWS has a 20 second idle timeout: # https://forums.aws.amazon.com/message.jspa?messageID=215367 # aiohttp default timeout is 30s so set something reasonable here connector_args = dict(keepalive_timeout=12) timeout = aiohttp.ClientTimeout(sock_connect=conn_timeout, sock_read=read_timeout) verify = self._get_verify_value(verify) ssl_context = None if client_cert: if isinstance(client_cert, str): key_file = None cert_file = client_cert elif isinstance(client_cert, tuple): cert_file, key_file = client_cert else: raise TypeError("client_cert must be str or tuple, not %s" % client_cert.__class__.__name__) ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) ssl_context.load_cert_chain(cert_file, key_file) elif isinstance(verify, (str, pathlib.Path)): ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=str(verify)) # TODO: add support for proxies_config connector = aiohttp.TCPConnector(limit=max_pool_connections, verify_ssl=bool(verify), ssl=ssl_context, **connector_args) aio_session = http_session_cls(connector=connector, timeout=timeout, skip_auto_headers={'CONTENT-TYPE'}, response_class=ClientResponseProxy, auto_decompress=False) return AioEndpoint(endpoint_url, endpoint_prefix=endpoint_prefix, event_emitter=self._event_emitter, response_parser_factory=response_parser_factory, http_session=aio_session, proxies=proxies)
async def _do_get_response(self, request, operation_model): try: # http request substituted too async one logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % \ (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: streaming = any([ operation_model.has_streaming_output, operation_model.has_event_stream_output ]) http_response = await self._request( request.method, request.url, request.headers, request.body, verify=self.verify_ssl, stream=streaming) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError( endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) parsed_response = parser.parse( response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None