def _build_call_parameters(self, args, param_dict):
service_name = self.service.cli_name
operation_name = self.operation.cli_name
for param in self.operation.params:
value = getattr(args, param.py_name)
if value is not None:
if not hasattr(param, 'no_paramfile'):
value = self._handle_param_file(value)
# Plugins can override the cli -> python conversion
# process for CLI args.
responses = self.session.emit('process-cli-arg.%s.%s' %
(service_name, operation_name),
param=param,
value=value,
service=self.service,
operation=self.operation)
override = first_non_none_response(responses)
if override is not None:
# A plugin supplied an alternate conversion,
# use it instead.
param_dict[param.py_name] = override
continue
# Otherwise fall back to our normal built in cli -> python
# conversion process.
if param.type == 'boolean' and not param.required and \
value is False:
# Don't include non-required boolean params whose
# values are False
continue
param_dict[param.py_name] = unpack_cli_arg(param, value)
def _convert_to_request_dict(self, api_params, operation_model):
# Given the API params provided by the user and the operation_model
# we can serialize the request to a request_dict.
operation_name = operation_model.name
# Emit an event that allows users to modify the parameters at the
# beginning of the method. It allows handlers to modify existing
# parameters or return a new set of parameters to use.
responses = self.meta.events.emit(
'provide-client-params.{endpoint_prefix}.{operation_name}'.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params, model=operation_model)
api_params = first_non_none_response(responses, default=api_params)
event_name = (
'before-parameter-build.{endpoint_prefix}.{operation_name}')
self.meta.events.emit(
event_name.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params, model=operation_model)
request_dict = self._serializer.serialize_to_request(
api_params, operation_model)
prepare_request_dict(request_dict, endpoint_url=self._endpoint.host,
user_agent=self._client_config.user_agent)
self.meta.events.emit(
'before-call.{endpoint_prefix}.{operation_name}'.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
model=operation_model, params=request_dict,
request_signer=self._request_signer
)
return request_dict
def _convert_to_request_dict(self, api_params, operation_model):
# Given the API params provided by the user and the operation_model
# we can serialize the request to a request_dict.
operation_name = operation_model.name
# Emit an event that allows users to modify the parameters at the
# beginning of the method. It allows handlers to modify existing
# parameters or return a new set of parameters to use.
responses = self.meta.events.emit(
'provide-client-params.{endpoint_prefix}.{operation_name}'.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params, model=operation_model)
api_params = first_non_none_response(responses, default=api_params)
event_name = (
'before-parameter-build.{endpoint_prefix}.{operation_name}')
self.meta.events.emit(
event_name.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params, model=operation_model)
request_dict = self._serializer.serialize_to_request(
api_params, operation_model)
prepare_request_dict(request_dict, endpoint_url=self._endpoint.host,
user_agent=self._client_config.user_agent)
self.meta.events.emit(
'before-call.{endpoint_prefix}.{operation_name}'.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
model=operation_model, params=request_dict,
request_signer=self._request_signer
)
return request_dict
def _emit_api_params(self, api_params, operation_model, context):
# Given the API params provided by the user and the operation_model
# we can serialize the request to a request_dict.
operation_name = operation_model.name
# Emit an event that allows users to modify the parameters at the
# beginning of the method. It allows handlers to modify existing
# parameters or return a new set of parameters to use.
responses = self.meta.events.emit(
'provide-client-params.{endpoint_prefix}.{operation_name}'.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params,
model=operation_model,
context=context)
api_params = first_non_none_response(responses, default=api_params)
event_name = (
'before-parameter-build.{endpoint_prefix}.{operation_name}')
self.meta.events.emit(event_name.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params,
model=operation_model,
context=context)
return api_params
def create_client(self,
service_name,
region_name,
is_secure=True,
endpoint_url=None,
verify=None,
credentials=None,
scoped_config=None,
api_version=None,
client_config=None):
responses = self._event_emitter.emit('choose-service-name',
service_name=service_name)
service_name = first_non_none_response(responses, default=service_name)
service_model = self._load_service_model(service_name, api_version)
cls = self._create_client_class(service_name, service_model)
endpoint_bridge = ClientEndpointBridge(
self._endpoint_resolver,
scoped_config,
client_config,
service_signing_name=service_model.metadata.get('signingName'))
client_args = self._get_client_args(service_model, region_name,
is_secure, endpoint_url, verify,
credentials, scoped_config,
client_config, endpoint_bridge)
service_client = cls(**client_args)
self._register_retries(service_client)
self._register_s3_events(service_client, endpoint_bridge, endpoint_url,
client_config, scoped_config)
return service_client
def _build_call_parameters(self, args, param_dict):
service_name = self.service.cli_name
operation_name = self.operation.cli_name
for param in self.operation.params:
value = getattr(args, param.py_name)
if value is not None:
# Plugins can override the cli -> python conversion
# process for CLI args.
responses = self.session.emit('process-cli-arg.%s.%s' % (
service_name, operation_name), param=param, value=value,
service=self.service, operation=self.operation)
override = first_non_none_response(responses)
if override is not None:
# A plugin supplied an alternate conversion,
# use it instead.
param_dict[param.py_name] = override
continue
# Otherwise fall back to our normal built in cli -> python
# conversion process.
if param.type == 'boolean' and not param.required and \
value is False:
# Don't include non-required boolean params whose
# values are False
continue
if not hasattr(param, 'no_paramfile'):
value = self._handle_param_file(value)
param_dict[param.py_name] = self._unpack_cli_arg(param, value)
def create_client(self, service_name, region_name, is_secure=True,
endpoint_url=None, verify=None,
credentials=None, scoped_config=None,
api_version=None,
client_config=None):
responses = self._event_emitter.emit(
'choose-service-name', service_name=service_name)
service_name = first_non_none_response(responses, default=service_name)
service_model = self._load_service_model(service_name, api_version)
cls = self._create_client_class(service_name, service_model)
endpoint_bridge = ClientEndpointBridge(
self._endpoint_resolver, scoped_config, client_config,
service_signing_name=service_model.metadata.get('signingName'))
client_args = self._get_client_args(
service_model, region_name, is_secure, endpoint_url,
verify, credentials, scoped_config, client_config, endpoint_bridge)
service_client = cls(**client_args)
self._register_retries(service_client)
self._register_s3_events(
service_client, endpoint_bridge, endpoint_url, client_config,
scoped_config)
self._register_endpoint_discovery(
service_client, endpoint_url, client_config
)
return service_client
def _needs_retry(self,
attempts,
operation_model,
request_dict,
response=None,
caught_exception=None):
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'needs-retry.%s.%s' % (service_id, operation_model.name)
responses = self._event_emitter.emit(event_name,
response=response,
endpoint=self,
operation=operation_model,
attempts=attempts,
caught_exception=caught_exception,
request_dict=request_dict)
handler_response = first_non_none_response(responses)
if handler_response is None:
return False
else:
# Request needs to be retried, and we need to sleep
# for the specified number of times.
logger.debug(
"Response received to retry, sleeping for "
"%s seconds", handler_response)
time.sleep(handler_response)
return True
def _needs_retry(self,
attempts,
operation_model,
request_dict,
response=None,
caught_exception=None):
event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix,
operation_model.name)
responses = self._event_emitter.emit(event_name,
response=response,
endpoint=self,
operation=operation_model,
attempts=attempts,
caught_exception=caught_exception,
request_dict=request_dict)
handler_response = first_non_none_response(responses)
if handler_response is None:
return False
else:
# Request needs to be retried, and we need to sleep
# for the specified number of times.
botocore.retryhandler.logger.debug(
"Response received to retry, "
"sleeping for %s seconds", handler_response)
yield from asyncio.sleep(handler_response, loop=self._loop)
return True
def test_decode_quoted_jsondoc(self):
event = self.session.create_event(
"after-parsed", "iam", "GetUserPolicy", "policyDocumentType", "PolicyDocument"
)
value = quote('{"foo":"bar"}')
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, {"foo": "bar"})
def test_first_non_none(self):
correct_value = 'correct_value'
wrong_value = 'wrong_value'
# The responses are tuples of (handler, response),
# and we don't care about the handler so we just use a value of
# None.
responses = [(None, None), (None, correct_value), (None, wrong_value)]
self.assertEqual(first_non_none_response(responses), correct_value)
def test_decode_jsondoc(self):
event = self.session.create_event(
"after-parsed", "cloudformation", "GetTemplate", "TemplateBody", "TemplateBody"
)
value = '{"foo":"bar"}'
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, {"foo": "bar"})
def test_get_console_output(self):
event = self.session.create_event('after-parsed', 'ec2',
'GetConsoleOutput',
'String', 'Output')
value = base64.b64encode(six.b('foobar')).decode('utf-8')
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, 'foobar')
def test_decode_jsondoc(self):
event = self.session.create_event('after-parsed', 'cloudformation',
'GetTemplate', 'TemplateBody',
'TemplateBody')
value = '{"foo":"bar"}'
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, {'foo': 'bar'})
def test_first_non_none(self):
correct_value = 'correct_value'
wrong_value = 'wrong_value'
# The responses are tuples of (handler, response),
# and we don't care about the handler so we just use a value of
# None.
responses = [(None, None), (None, correct_value), (None, wrong_value)]
self.assertEqual(first_non_none_response(responses), correct_value)
def test_get_console_output(self):
event = self.session.create_event('after-parsed', 'ec2',
'GetConsoleOutput', 'String',
'Output')
value = base64.b64encode(six.b('foobar')).decode('utf-8')
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, 'foobar')
def test_decode_quoted_jsondoc(self):
event = self.session.create_event('after-parsed', 'iam',
'GetUserPolicy',
'policyDocumentType',
'PolicyDocument')
value = quote('{"foo":"bar"}')
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, {'foo': 'bar'})
def test_decode_quoted_jsondoc(self):
event = self.session.create_event('after-parsed', 'iam',
'GetUserPolicy',
'policyDocumentType',
'PolicyDocument')
value = quote('{"foo":"bar"}')
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, {'foo': 'bar'})
def test_decode_jsondoc(self):
event = self.session.create_event('after-parsed', 'cloudformation',
'GetTemplate',
'TemplateBody',
'TemplateBody')
value = '{"foo":"bar"}'
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, {'foo':'bar'})
async def _do_get_response(self, request, operation_model):
try:
logger.debug("Sending http request: %s", request)
history_recorder.record('HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % (
service_id, operation_model.name)
responses = await self._event_emitter.emit(event_name,
request=request)
http_response = first_non_none_response(responses)
if http_response is None:
http_response = await self._send(request)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(
endpoint_url=request.url, request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
if asyncio.iscoroutinefunction(parser.parse):
parsed_response = await parser.parse(
response_dict, operation_model.output_shape)
else:
parsed_response = parser.parse(
response_dict, operation_model.output_shape)
if http_response.status_code >= 300:
await self._add_modeled_error_fields(
response_dict, parsed_response,
operation_model, parser,
)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def emit_event(self, tag, shape, value):
if 'shape_name' in shape:
event = self.session.create_event(
'after-parsed', self.operation.service.endpoint_prefix,
self.operation.name, shape['shape_name'], tag)
rv = first_non_none_response(
self.session.emit(event, shape=shape, value=value), None)
if rv:
value = rv
return value
def emit_event(self, tag, shape, value):
if 'shape_name' in shape:
event = self.session.create_event(
'after-parsed', self.operation.service.endpoint_prefix,
self.operation.name, shape['shape_name'], tag)
rv = first_non_none_response(self.session.emit(event,
shape=shape,
value=value),
None)
if rv:
value = rv
return value
def _do_get_response(self, request, operation_model):
try:
logger.debug("Sending http request: %s", request)
history_recorder.record(
'HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % (service_id,
operation_model.name)
responses = self._event_emitter.emit(event_name, request=request)
http_response = first_non_none_response(responses)
if http_response is None:
http_response = self._send(request)
except HTTPClientError as e:
return (None, e)
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return (None, e)
# This returns the http_response and the parsed_data.
response_dict = convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
parsed_response = parser.parse(response_dict,
operation_model.output_shape)
# Do a second parsing pass to pick up on any modeled error fields
# NOTE: Ideally, we would push this down into the parser classes but
# they currently have no reference to the operation or service model
# The parsers should probably take the operation model instead of
# output shape but we can't change that now
if http_response.status_code >= 300:
self._add_modeled_error_fields(
response_dict,
parsed_response,
operation_model,
parser,
)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def _unpack_argument(self, value):
if not hasattr(self.argument_object, 'no_paramfile'):
value = self._handle_param_file(value)
service_name = self.operation_object.service.endpoint_prefix
operation_name = xform_name(self.operation_object.name, '-')
responses = self._emit('process-cli-arg.%s.%s' % (
service_name, operation_name), param=self.argument_object,
value=value,
operation=self.operation_object)
override = first_non_none_response(responses)
if override is not None:
# A plugin supplied an alternate conversion,
# use it instead.
return override
else:
# Fall back to the default arg processing.
return unpack_cli_arg(self.argument_object, value)
def _unpack_argument(self, value):
if not hasattr(self.argument_object, 'no_paramfile'):
value = self._handle_param_file(value)
service_name = self.operation_object.service.endpoint_prefix
operation_name = xform_name(self.operation_object.name, '-')
responses = self._emit('process-cli-arg.%s.%s' % (
service_name, operation_name), param=self.argument_object,
value=value,
operation=self.operation_object)
override = first_non_none_response(responses)
if override is not None:
# A plugin supplied an alternate conversion,
# use it instead.
return override
else:
# Fall back to the default arg processing.
return unpack_cli_arg(self.argument_object, value)
def _needs_retry(self, attempts, operation_model, response=None,
caught_exception=None):
event_name = 'needs-retry.%s.%s' % (self._endpoint_prefix,
operation_model.name)
responses = self._event_emitter.emit(
event_name, response=response, endpoint=self,
operation=operation_model, attempts=attempts,
caught_exception=caught_exception)
handler_response = first_non_none_response(responses)
if handler_response is None:
return False
else:
# Request needs to be retried, and we need to sleep
# for the specified number of times.
logger.debug("Response received to retry, sleeping for "
"%s seconds", handler_response)
time.sleep(handler_response)
return True
def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
logger.debug("Sending http request: %s", request)
history_recorder.record(
'HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % (service_id,
operation_model.name)
responses = self._event_emitter.emit(event_name, request=request)
http_response = first_non_none_response(responses)
if http_response is None:
http_response = self._send(request)
except HTTPClientError as e:
return (None, e)
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return (None, e)
# This returns the http_response and the parsed_data.
response_dict = convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
parsed_response = parser.parse(response_dict,
operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def _do_request(self, action, op_name, **kwargs):
api_params = kwargs
operation_model = self.meta.service_model.operation_model(
self.meta.method_to_api_mapping[op_name])
request_context = {
'client_region': self.meta.region_name,
'client_config': self.meta.config,
'has_streaming_input': operation_model.has_streaming_input,
'auth_type': operation_model.auth_type,
}
# Copy boto3's events so that the resource works properly
# Emit an event that allows users to modify the parameters at the
# beginning of the method. It allows handlers to modify existing
# parameters or return a new set of parameters to use.
responses = self.meta.events.emit(
'provide-client-params.dynamodb.{operation_name}'.format(
operation_name=operation_model.name),
params=api_params,
model=operation_model,
context=request_context)
api_params = first_non_none_response(responses, default=api_params)
self.meta.events.emit(
'before-parameter-build.dynamodb.{operation_name}'.format(
operation_name=operation_model.name),
params=api_params,
model=operation_model,
context=request_context)
response = action(**api_params)
# Ignore the HTTP response and just handle the parsed response
self.meta.events.emit('after-call.dynamodb.{operation_name}'.format(
operation_name=operation_model.name),
http_response=None,
parsed=response,
model=operation_model,
context=request_context)
return response
def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
logger.debug("Sending http request: %s", request)
history_recorder.record('HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % (service_id, operation_model.name)
responses = self._event_emitter.emit(event_name, request=request)
http_response = first_non_none_response(responses)
if http_response is None:
http_response = self._send(request)
except HTTPClientError as e:
return (None, e)
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return (None, e)
# This returns the http_response and the parsed_data.
response_dict = convert_to_response_dict(http_response, operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
parsed_response = parser.parse(
response_dict, operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def _emit_api_params(self, api_params, operation_model, context):
# Given the API params provided by the user and the operation_model
# we can serialize the request to a request_dict.
operation_name = operation_model.name
# Emit an event that allows users to modify the parameters at the
# beginning of the method. It allows handlers to modify existing
# parameters or return a new set of parameters to use.
responses = self.meta.events.emit(
'provide-client-params.{endpoint_prefix}.{operation_name}'.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params, model=operation_model, context=context)
api_params = first_non_none_response(responses, default=api_params)
event_name = (
'before-parameter-build.{endpoint_prefix}.{operation_name}')
self.meta.events.emit(
event_name.format(
endpoint_prefix=self._service_model.endpoint_prefix,
operation_name=operation_name),
params=api_params, model=operation_model, context=context)
return api_params
def _emit_first_response(self, name, **kwargs):
responses = self._emit(name, **kwargs)
return first_non_none_response(responses)
def test_default_value_if_non_none_found(self):
responses = [(None, None), (None, None)]
# If no response is found and a default value is passed in, it will
# be returned.
self.assertEqual(
first_non_none_response(responses, default='notfound'), 'notfound')
def test_get_console_output(self):
event = self.session.create_event("after-parsed", "ec2", "GetConsoleOutput", "String", "Output")
value = base64.b64encode(six.b("foobar")).decode("utf-8")
rv = self.session.emit(event, shape={}, value=value)
converted_value = first_non_none_response(rv)
self.assertEqual(converted_value, "foobar")
def test_all_none(self):
self.assertIsNone(first_non_none_response([]))
def emit_first_non_none_response(self, event_name, **kwargs):
responses = self._events.emit(event_name, **kwargs)
return first_non_none_response(responses)
def emit_first_non_none_response(self, event_name, **kwargs):
responses = self._events.emit(event_name, **kwargs)
return first_non_none_response(responses)
def _emit_first_response(self, name, **kwargs):
responses = self._emit(name, **kwargs)
return first_non_none_response(responses)
def test_default_value_if_non_none_found(self):
responses = [(None, None), (None, None)]
# If no response is found and a default value is passed in, it will
# be returned.
self.assertEqual(
first_non_none_response(responses, default='notfound'), 'notfound')
async def _do_get_response(self, request, operation_model):
try:
# http request substituted too async one
logger.debug("Sending http request: %s", request)
history_recorder.record('HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % \
(service_id, operation_model.name)
responses = self._event_emitter.emit(event_name, request=request)
http_response = first_non_none_response(responses)
if http_response is None:
streaming = any([
operation_model.has_streaming_output,
operation_model.has_event_stream_output
])
http_response = await self._request(
request.method, request.url, request.headers, request.body,
verify=self.verify_ssl,
stream=streaming)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(
endpoint_url=request.url, request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
parsed_response = parser.parse(
response_dict, operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
def test_all_none(self):
self.assertIsNone(first_non_none_response([]))
def _make_api_call(self, operation_name, operation_kwargs):
"""
This private method is here for two reasons:
1. It's faster to avoid using botocore's response parsing
2. It provides a place to monkey patch HTTP requests for unit testing
"""
operation_model = self.client._service_model.operation_model(operation_name)
request_dict = self.client._convert_to_request_dict(
operation_kwargs,
operation_model,
)
for i in range(0, self._max_retry_attempts_exception + 1):
attempt_number = i + 1
is_last_attempt_for_exceptions = i == self._max_retry_attempts_exception
http_response = None
prepared_request = None
try:
if prepared_request is not None:
# If there is a stream associated with the request, we need
# to reset it before attempting to send the request again.
# This will ensure that we resend the entire contents of the
# body.
prepared_request.reset_stream()
# Create a new request for each retry (including a new signature).
prepared_request = self._create_prepared_request(request_dict, operation_model)
# Implement the before-send event from botocore
event_name = 'before-send.dynamodb.{}'.format(operation_model.name)
event_responses = self.client._endpoint._event_emitter.emit(event_name, request=prepared_request)
event_response = first_non_none_response(event_responses)
if event_response is None:
http_response = self.client._endpoint.http_session.send(prepared_request)
else:
http_response = event_response
is_last_attempt_for_exceptions = True # don't retry if we have an event response
# json.loads accepts bytes in >= 3.6.0
if sys.version_info < (3, 6, 0):
data = json.loads(http_response.text)
else:
data = json.loads(http_response.content)
except (ValueError, botocore.exceptions.HTTPClientError, botocore.exceptions.ConnectionError) as e:
if is_last_attempt_for_exceptions:
log.debug('Reached the maximum number of retry attempts: %s', attempt_number)
if http_response:
e.args += (http_response.text,)
raise
else:
# No backoff for fast-fail exceptions that likely failed at the frontend
log.debug(
'Retry needed for (%s) after attempt %s, retryable %s caught: %s',
operation_name,
attempt_number,
e.__class__.__name__,
e
)
continue
status_code = http_response.status_code
headers = http_response.headers
if status_code >= 300:
# Extract error code from __type
code = data.get('__type', '')
if '#' in code:
code = code.rsplit('#', 1)[1]
botocore_expected_format = {'Error': {'Message': data.get('message', ''), 'Code': code}}
verbose_properties = {
'request_id': headers.get('x-amzn-RequestId')
}
if 'RequestItems' in operation_kwargs:
# Batch operations can hit multiple tables, report them comma separated
verbose_properties['table_name'] = ','.join(operation_kwargs['RequestItems'])
else:
verbose_properties['table_name'] = operation_kwargs.get('TableName')
try:
raise VerboseClientError(botocore_expected_format, operation_name, verbose_properties)
except VerboseClientError as e:
if is_last_attempt_for_exceptions:
log.debug('Reached the maximum number of retry attempts: %s', attempt_number)
raise
elif status_code < 500 and code != 'ProvisionedThroughputExceededException':
# We don't retry on a ConditionalCheckFailedException or other 4xx (except for
# throughput related errors) because we assume they will fail in perpetuity.
# Retrying when there is already contention could cause other problems
# in part due to unnecessary consumption of throughput.
raise
else:
# We use fully-jittered exponentially-backed-off retries:
# https://www.awsarchitectureblog.com/2015/03/backoff.html
sleep_time_ms = random.randint(0, self._base_backoff_ms * (2 ** i))
log.debug(
'Retry with backoff needed for (%s) after attempt %s,'
'sleeping for %s milliseconds, retryable %s caught: %s',
operation_name,
attempt_number,
sleep_time_ms,
e.__class__.__name__,
e
)
time.sleep(sleep_time_ms / 1000.0)
continue
return self._handle_binary_attributes(data)
async def _get_response(self, request, operation_model, attempts):
# This will return a tuple of (success_response, exception)
# and success_response is itself a tuple of
# (http_response, parsed_dict).
# If an exception occurs then the success_response is None.
# If no exception occurs then exception is None.
try:
# http request substituted too async one
logger.debug("Sending http request: %s", request)
history_recorder.record(
'HTTP_REQUEST', {
'method': request.method,
'headers': request.headers,
'streaming': operation_model.has_streaming_input,
'url': request.url,
'body': request.body
})
service_id = operation_model.service_model.service_id.hyphenize()
event_name = 'before-send.%s.%s' % \
(service_id, operation_model.name)
responses = self._event_emitter.emit(event_name, request=request)
http_response = first_non_none_response(responses)
if http_response is None:
streaming = any([
operation_model.has_streaming_output,
operation_model.has_event_stream_output
])
http_response = await self._request(request.method,
request.url,
request.headers,
request.body,
verify=self.verify_ssl,
stream=streaming)
except aiohttp.ClientConnectionError as e:
e.request = request # botocore expects the request property
# For a connection error, if it looks like it's a DNS
# lookup issue, 99% of the time this is due to a misconfigured
# region/endpoint so we'll raise a more specific error message
# to help users.
logger.debug("ConnectionError received when sending HTTP request.",
exc_info=True)
return None, e
except aiohttp.http_exceptions.BadStatusLine:
better_exception = ConnectionClosedError(endpoint_url=request.url,
request=request)
return None, better_exception
except Exception as e:
logger.debug("Exception received when sending HTTP request.",
exc_info=True)
return None, e
# This returns the http_response and the parsed_data.
response_dict = await convert_to_response_dict(http_response,
operation_model)
http_response_record_dict = response_dict.copy()
http_response_record_dict['streaming'] = \
operation_model.has_streaming_output
history_recorder.record('HTTP_RESPONSE', http_response_record_dict)
protocol = operation_model.metadata['protocol']
parser = self._response_parser_factory.create_parser(protocol)
parsed_response = parser.parse(response_dict,
operation_model.output_shape)
history_recorder.record('PARSED_RESPONSE', parsed_response)
return (http_response, parsed_response), None
