def _build_call_parameters(self, args, param_dict): service_name = self.service.cli_name operation_name = self.operation.cli_name for param in self.operation.params: value = getattr(args, param.py_name) if value is not None: if not hasattr(param, 'no_paramfile'): value = self._handle_param_file(value) # Plugins can override the cli -> python conversion # process for CLI args. responses = self.session.emit('process-cli-arg.%s.%s' % (service_name, operation_name), param=param, value=value, service=self.service, operation=self.operation) override = first_non_none_response(responses) if override is not None: # A plugin supplied an alternate conversion, # use it instead. param_dict[param.py_name] = override continue # Otherwise fall back to our normal built in cli -> python # conversion process. if param.type == 'boolean' and not param.required and \ value is False: # Don't include non-required boolean params whose # values are False continue param_dict[param.py_name] = unpack_cli_arg(param, value)
def _convert_to_request_dict(self, api_params, operation_model): # Given the API params provided by the user and the operation_model # we can serialize the request to a request_dict. operation_name = operation_model.name # Emit an event that allows users to modify the parameters at the # beginning of the method. It allows handlers to modify existing # parameters or return a new set of parameters to use. responses = self.meta.events.emit( 'provide-client-params.{endpoint_prefix}.{operation_name}'.format( endpoint_prefix=self._service_model.endpoint_prefix, operation_name=operation_name), params=api_params, model=operation_model) api_params = first_non_none_response(responses, default=api_params) event_name = ( 'before-parameter-build.{endpoint_prefix}.{operation_name}') self.meta.events.emit( event_name.format( endpoint_prefix=self._service_model.endpoint_prefix, operation_name=operation_name), params=api_params, model=operation_model) request_dict = self._serializer.serialize_to_request( api_params, operation_model) prepare_request_dict(request_dict, endpoint_url=self._endpoint.host, user_agent=self._client_config.user_agent) self.meta.events.emit( 'before-call.{endpoint_prefix}.{operation_name}'.format( endpoint_prefix=self._service_model.endpoint_prefix, operation_name=operation_name), model=operation_model, params=request_dict, request_signer=self._request_signer ) return request_dict
def _emit_api_params(self, api_params, operation_model, context): # Given the API params provided by the user and the operation_model # we can serialize the request to a request_dict. operation_name = operation_model.name # Emit an event that allows users to modify the parameters at the # beginning of the method. It allows handlers to modify existing # parameters or return a new set of parameters to use. responses = self.meta.events.emit( 'provide-client-params.{endpoint_prefix}.{operation_name}'.format( endpoint_prefix=self._service_model.endpoint_prefix, operation_name=operation_name), params=api_params, model=operation_model, context=context) api_params = first_non_none_response(responses, default=api_params) event_name = ( 'before-parameter-build.{endpoint_prefix}.{operation_name}') self.meta.events.emit(event_name.format( endpoint_prefix=self._service_model.endpoint_prefix, operation_name=operation_name), params=api_params, model=operation_model, context=context) return api_params
def create_client(self, service_name, region_name, is_secure=True, endpoint_url=None, verify=None, credentials=None, scoped_config=None, api_version=None, client_config=None): responses = self._event_emitter.emit('choose-service-name', service_name=service_name) service_name = first_non_none_response(responses, default=service_name) service_model = self._load_service_model(service_name, api_version) cls = self._create_client_class(service_name, service_model) endpoint_bridge = ClientEndpointBridge( self._endpoint_resolver, scoped_config, client_config, service_signing_name=service_model.metadata.get('signingName')) client_args = self._get_client_args(service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge) service_client = cls(**client_args) self._register_retries(service_client) self._register_s3_events(service_client, endpoint_bridge, endpoint_url, client_config, scoped_config) return service_client
def _build_call_parameters(self, args, param_dict): service_name = self.service.cli_name operation_name = self.operation.cli_name for param in self.operation.params: value = getattr(args, param.py_name) if value is not None: # Plugins can override the cli -> python conversion # process for CLI args. responses = self.session.emit('process-cli-arg.%s.%s' % ( service_name, operation_name), param=param, value=value, service=self.service, operation=self.operation) override = first_non_none_response(responses) if override is not None: # A plugin supplied an alternate conversion, # use it instead. param_dict[param.py_name] = override continue # Otherwise fall back to our normal built in cli -> python # conversion process. if param.type == 'boolean' and not param.required and \ value is False: # Don't include non-required boolean params whose # values are False continue if not hasattr(param, 'no_paramfile'): value = self._handle_param_file(value) param_dict[param.py_name] = self._unpack_cli_arg(param, value)
def create_client(self, service_name, region_name, is_secure=True, endpoint_url=None, verify=None, credentials=None, scoped_config=None, api_version=None, client_config=None): responses = self._event_emitter.emit( 'choose-service-name', service_name=service_name) service_name = first_non_none_response(responses, default=service_name) service_model = self._load_service_model(service_name, api_version) cls = self._create_client_class(service_name, service_model) endpoint_bridge = ClientEndpointBridge( self._endpoint_resolver, scoped_config, client_config, service_signing_name=service_model.metadata.get('signingName')) client_args = self._get_client_args( service_model, region_name, is_secure, endpoint_url, verify, credentials, scoped_config, client_config, endpoint_bridge) service_client = cls(**client_args) self._register_retries(service_client) self._register_s3_events( service_client, endpoint_bridge, endpoint_url, client_config, scoped_config) self._register_endpoint_discovery( service_client, endpoint_url, client_config ) return service_client
def _needs_retry(self, attempts, operation_model, request_dict, response=None, caught_exception=None): service_id = operation_model.service_model.service_id.hyphenize() event_name = 'needs-retry.%s.%s' % (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, response=response, endpoint=self, operation=operation_model, attempts=attempts, caught_exception=caught_exception, request_dict=request_dict) handler_response = first_non_none_response(responses) if handler_response is None: return False else: # Request needs to be retried, and we need to sleep # for the specified number of times. logger.debug( "Response received to retry, sleeping for " "%s seconds", handler_response) time.sleep(handler_response) return True
def _needs_retry(self, attempts, operation_model, request_dict, response=None, caught_exception=None): event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix, operation_model.name) responses = self._event_emitter.emit(event_name, response=response, endpoint=self, operation=operation_model, attempts=attempts, caught_exception=caught_exception, request_dict=request_dict) handler_response = first_non_none_response(responses) if handler_response is None: return False else: # Request needs to be retried, and we need to sleep # for the specified number of times. botocore.retryhandler.logger.debug( "Response received to retry, " "sleeping for %s seconds", handler_response) yield from asyncio.sleep(handler_response, loop=self._loop) return True
def test_decode_quoted_jsondoc(self): event = self.session.create_event( "after-parsed", "iam", "GetUserPolicy", "policyDocumentType", "PolicyDocument" ) value = quote('{"foo":"bar"}') rv = self.session.emit(event, shape={}, value=value) converted_value = first_non_none_response(rv) self.assertEqual(converted_value, {"foo": "bar"})
def test_first_non_none(self): correct_value = 'correct_value' wrong_value = 'wrong_value' # The responses are tuples of (handler, response), # and we don't care about the handler so we just use a value of # None. responses = [(None, None), (None, correct_value), (None, wrong_value)] self.assertEqual(first_non_none_response(responses), correct_value)
def test_decode_jsondoc(self): event = self.session.create_event( "after-parsed", "cloudformation", "GetTemplate", "TemplateBody", "TemplateBody" ) value = '{"foo":"bar"}' rv = self.session.emit(event, shape={}, value=value) converted_value = first_non_none_response(rv) self.assertEqual(converted_value, {"foo": "bar"})
def test_get_console_output(self): event = self.session.create_event('after-parsed', 'ec2', 'GetConsoleOutput', 'String', 'Output') value = base64.b64encode(six.b('foobar')).decode('utf-8') rv = self.session.emit(event, shape={}, value=value) converted_value = first_non_none_response(rv) self.assertEqual(converted_value, 'foobar')
def test_decode_jsondoc(self): event = self.session.create_event('after-parsed', 'cloudformation', 'GetTemplate', 'TemplateBody', 'TemplateBody') value = '{"foo":"bar"}' rv = self.session.emit(event, shape={}, value=value) converted_value = first_non_none_response(rv) self.assertEqual(converted_value, {'foo': 'bar'})
def test_decode_quoted_jsondoc(self): event = self.session.create_event('after-parsed', 'iam', 'GetUserPolicy', 'policyDocumentType', 'PolicyDocument') value = quote('{"foo":"bar"}') rv = self.session.emit(event, shape={}, value=value) converted_value = first_non_none_response(rv) self.assertEqual(converted_value, {'foo': 'bar'})
def test_decode_jsondoc(self): event = self.session.create_event('after-parsed', 'cloudformation', 'GetTemplate', 'TemplateBody', 'TemplateBody') value = '{"foo":"bar"}' rv = self.session.emit(event, shape={}, value=value) converted_value = first_non_none_response(rv) self.assertEqual(converted_value, {'foo':'bar'})
async def _do_get_response(self, request, operation_model): try: logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % ( service_id, operation_model.name) responses = await self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: http_response = await self._send(request) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError( endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) if asyncio.iscoroutinefunction(parser.parse): parsed_response = await parser.parse( response_dict, operation_model.output_shape) else: parsed_response = parser.parse( response_dict, operation_model.output_shape) if http_response.status_code >= 300: await self._add_modeled_error_fields( response_dict, parsed_response, operation_model, parser, ) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def emit_event(self, tag, shape, value): if 'shape_name' in shape: event = self.session.create_event( 'after-parsed', self.operation.service.endpoint_prefix, self.operation.name, shape['shape_name'], tag) rv = first_non_none_response( self.session.emit(event, shape=shape, value=value), None) if rv: value = rv return value
def emit_event(self, tag, shape, value): if 'shape_name' in shape: event = self.session.create_event( 'after-parsed', self.operation.service.endpoint_prefix, self.operation.name, shape['shape_name'], tag) rv = first_non_none_response(self.session.emit(event, shape=shape, value=value), None) if rv: value = rv return value
def _do_get_response(self, request, operation_model): try: logger.debug("Sending http request: %s", request) history_recorder.record( 'HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: http_response = self._send(request) except HTTPClientError as e: return (None, e) except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return (None, e) # This returns the http_response and the parsed_data. response_dict = convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) parsed_response = parser.parse(response_dict, operation_model.output_shape) # Do a second parsing pass to pick up on any modeled error fields # NOTE: Ideally, we would push this down into the parser classes but # they currently have no reference to the operation or service model # The parsers should probably take the operation model instead of # output shape but we can't change that now if http_response.status_code >= 300: self._add_modeled_error_fields( response_dict, parsed_response, operation_model, parser, ) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def _unpack_argument(self, value): if not hasattr(self.argument_object, 'no_paramfile'): value = self._handle_param_file(value) service_name = self.operation_object.service.endpoint_prefix operation_name = xform_name(self.operation_object.name, '-') responses = self._emit('process-cli-arg.%s.%s' % ( service_name, operation_name), param=self.argument_object, value=value, operation=self.operation_object) override = first_non_none_response(responses) if override is not None: # A plugin supplied an alternate conversion, # use it instead. return override else: # Fall back to the default arg processing. return unpack_cli_arg(self.argument_object, value)
def _needs_retry(self, attempts, operation_model, response=None, caught_exception=None): event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix, operation_model.name) responses = self._event_emitter.emit( event_name, response=response, endpoint=self, operation=operation_model, attempts=attempts, caught_exception=caught_exception) handler_response = first_non_none_response(responses) if handler_response is None: return False else: # Request needs to be retried, and we need to sleep # for the specified number of times. logger.debug("Response received to retry, sleeping for " "%s seconds", handler_response) time.sleep(handler_response) return True
def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: logger.debug("Sending http request: %s", request) history_recorder.record( 'HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: http_response = self._send(request) except HTTPClientError as e: return (None, e) except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return (None, e) # This returns the http_response and the parsed_data. response_dict = convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) parsed_response = parser.parse(response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def _do_request(self, action, op_name, **kwargs): api_params = kwargs operation_model = self.meta.service_model.operation_model( self.meta.method_to_api_mapping[op_name]) request_context = { 'client_region': self.meta.region_name, 'client_config': self.meta.config, 'has_streaming_input': operation_model.has_streaming_input, 'auth_type': operation_model.auth_type, } # Copy boto3's events so that the resource works properly # Emit an event that allows users to modify the parameters at the # beginning of the method. It allows handlers to modify existing # parameters or return a new set of parameters to use. responses = self.meta.events.emit( 'provide-client-params.dynamodb.{operation_name}'.format( operation_name=operation_model.name), params=api_params, model=operation_model, context=request_context) api_params = first_non_none_response(responses, default=api_params) self.meta.events.emit( 'before-parameter-build.dynamodb.{operation_name}'.format( operation_name=operation_model.name), params=api_params, model=operation_model, context=request_context) response = action(**api_params) # Ignore the HTTP response and just handle the parsed response self.meta.events.emit('after-call.dynamodb.{operation_name}'.format( operation_name=operation_model.name), http_response=None, parsed=response, model=operation_model, context=request_context) return response
def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: http_response = self._send(request) except HTTPClientError as e: return (None, e) except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return (None, e) # This returns the http_response and the parsed_data. response_dict = convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) parsed_response = parser.parse( response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def _emit_api_params(self, api_params, operation_model, context): # Given the API params provided by the user and the operation_model # we can serialize the request to a request_dict. operation_name = operation_model.name # Emit an event that allows users to modify the parameters at the # beginning of the method. It allows handlers to modify existing # parameters or return a new set of parameters to use. responses = self.meta.events.emit( 'provide-client-params.{endpoint_prefix}.{operation_name}'.format( endpoint_prefix=self._service_model.endpoint_prefix, operation_name=operation_name), params=api_params, model=operation_model, context=context) api_params = first_non_none_response(responses, default=api_params) event_name = ( 'before-parameter-build.{endpoint_prefix}.{operation_name}') self.meta.events.emit( event_name.format( endpoint_prefix=self._service_model.endpoint_prefix, operation_name=operation_name), params=api_params, model=operation_model, context=context) return api_params
def _emit_first_response(self, name, **kwargs): responses = self._emit(name, **kwargs) return first_non_none_response(responses)
def test_default_value_if_non_none_found(self): responses = [(None, None), (None, None)] # If no response is found and a default value is passed in, it will # be returned. self.assertEqual( first_non_none_response(responses, default='notfound'), 'notfound')
def test_get_console_output(self): event = self.session.create_event("after-parsed", "ec2", "GetConsoleOutput", "String", "Output") value = base64.b64encode(six.b("foobar")).decode("utf-8") rv = self.session.emit(event, shape={}, value=value) converted_value = first_non_none_response(rv) self.assertEqual(converted_value, "foobar")
def test_all_none(self): self.assertIsNone(first_non_none_response([]))
def emit_first_non_none_response(self, event_name, **kwargs): responses = self._events.emit(event_name, **kwargs) return first_non_none_response(responses)
async def _do_get_response(self, request, operation_model): try: # http request substituted too async one logger.debug("Sending http request: %s", request) history_recorder.record('HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % \ (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: streaming = any([ operation_model.has_streaming_output, operation_model.has_event_stream_output ]) http_response = await self._request( request.method, request.url, request.headers, request.body, verify=self.verify_ssl, stream=streaming) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError( endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) parsed_response = parser.parse( response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None
def _make_api_call(self, operation_name, operation_kwargs): """ This private method is here for two reasons: 1. It's faster to avoid using botocore's response parsing 2. It provides a place to monkey patch HTTP requests for unit testing """ operation_model = self.client._service_model.operation_model(operation_name) request_dict = self.client._convert_to_request_dict( operation_kwargs, operation_model, ) for i in range(0, self._max_retry_attempts_exception + 1): attempt_number = i + 1 is_last_attempt_for_exceptions = i == self._max_retry_attempts_exception http_response = None prepared_request = None try: if prepared_request is not None: # If there is a stream associated with the request, we need # to reset it before attempting to send the request again. # This will ensure that we resend the entire contents of the # body. prepared_request.reset_stream() # Create a new request for each retry (including a new signature). prepared_request = self._create_prepared_request(request_dict, operation_model) # Implement the before-send event from botocore event_name = 'before-send.dynamodb.{}'.format(operation_model.name) event_responses = self.client._endpoint._event_emitter.emit(event_name, request=prepared_request) event_response = first_non_none_response(event_responses) if event_response is None: http_response = self.client._endpoint.http_session.send(prepared_request) else: http_response = event_response is_last_attempt_for_exceptions = True # don't retry if we have an event response # json.loads accepts bytes in >= 3.6.0 if sys.version_info < (3, 6, 0): data = json.loads(http_response.text) else: data = json.loads(http_response.content) except (ValueError, botocore.exceptions.HTTPClientError, botocore.exceptions.ConnectionError) as e: if is_last_attempt_for_exceptions: log.debug('Reached the maximum number of retry attempts: %s', attempt_number) if http_response: e.args += (http_response.text,) raise else: # No backoff for fast-fail exceptions that likely failed at the frontend log.debug( 'Retry needed for (%s) after attempt %s, retryable %s caught: %s', operation_name, attempt_number, e.__class__.__name__, e ) continue status_code = http_response.status_code headers = http_response.headers if status_code >= 300: # Extract error code from __type code = data.get('__type', '') if '#' in code: code = code.rsplit('#', 1)[1] botocore_expected_format = {'Error': {'Message': data.get('message', ''), 'Code': code}} verbose_properties = { 'request_id': headers.get('x-amzn-RequestId') } if 'RequestItems' in operation_kwargs: # Batch operations can hit multiple tables, report them comma separated verbose_properties['table_name'] = ','.join(operation_kwargs['RequestItems']) else: verbose_properties['table_name'] = operation_kwargs.get('TableName') try: raise VerboseClientError(botocore_expected_format, operation_name, verbose_properties) except VerboseClientError as e: if is_last_attempt_for_exceptions: log.debug('Reached the maximum number of retry attempts: %s', attempt_number) raise elif status_code < 500 and code != 'ProvisionedThroughputExceededException': # We don't retry on a ConditionalCheckFailedException or other 4xx (except for # throughput related errors) because we assume they will fail in perpetuity. # Retrying when there is already contention could cause other problems # in part due to unnecessary consumption of throughput. raise else: # We use fully-jittered exponentially-backed-off retries: # https://www.awsarchitectureblog.com/2015/03/backoff.html sleep_time_ms = random.randint(0, self._base_backoff_ms * (2 ** i)) log.debug( 'Retry with backoff needed for (%s) after attempt %s,' 'sleeping for %s milliseconds, retryable %s caught: %s', operation_name, attempt_number, sleep_time_ms, e.__class__.__name__, e ) time.sleep(sleep_time_ms / 1000.0) continue return self._handle_binary_attributes(data)
async def _get_response(self, request, operation_model, attempts): # This will return a tuple of (success_response, exception) # and success_response is itself a tuple of # (http_response, parsed_dict). # If an exception occurs then the success_response is None. # If no exception occurs then exception is None. try: # http request substituted too async one logger.debug("Sending http request: %s", request) history_recorder.record( 'HTTP_REQUEST', { 'method': request.method, 'headers': request.headers, 'streaming': operation_model.has_streaming_input, 'url': request.url, 'body': request.body }) service_id = operation_model.service_model.service_id.hyphenize() event_name = 'before-send.%s.%s' % \ (service_id, operation_model.name) responses = self._event_emitter.emit(event_name, request=request) http_response = first_non_none_response(responses) if http_response is None: streaming = any([ operation_model.has_streaming_output, operation_model.has_event_stream_output ]) http_response = await self._request(request.method, request.url, request.headers, request.body, verify=self.verify_ssl, stream=streaming) except aiohttp.ClientConnectionError as e: e.request = request # botocore expects the request property # For a connection error, if it looks like it's a DNS # lookup issue, 99% of the time this is due to a misconfigured # region/endpoint so we'll raise a more specific error message # to help users. logger.debug("ConnectionError received when sending HTTP request.", exc_info=True) return None, e except aiohttp.http_exceptions.BadStatusLine: better_exception = ConnectionClosedError(endpoint_url=request.url, request=request) return None, better_exception except Exception as e: logger.debug("Exception received when sending HTTP request.", exc_info=True) return None, e # This returns the http_response and the parsed_data. response_dict = await convert_to_response_dict(http_response, operation_model) http_response_record_dict = response_dict.copy() http_response_record_dict['streaming'] = \ operation_model.has_streaming_output history_recorder.record('HTTP_RESPONSE', http_response_record_dict) protocol = operation_model.metadata['protocol'] parser = self._response_parser_factory.create_parser(protocol) parsed_response = parser.parse(response_dict, operation_model.output_shape) history_recorder.record('PARSED_RESPONSE', parsed_response) return (http_response, parsed_response), None