def document_batch_action(
section,
resource_name,
event_emitter,
batch_action_model,
service_model,
collection_model,
include_signature=True,
):
"""Documents a collection's batch action
:param section: The section to write to
:param resource_name: The name of the resource
:param action_name: The name of collection action. Currently only
can be all, filter, limit, or page_size
:param event_emitter: The event emitter to use to emit events
:param batch_action_model: The model of the batch action
:param collection_model: The model of the collection
:param service_model: The model of the service
:param include_signature: Whether or not to include the signature.
It is useful for generating docstrings.
"""
operation_model = service_model.operation_model(
batch_action_model.request.operation
)
ignore_params = get_resource_ignore_params(
batch_action_model.request.params
)
example_return_value = 'response'
if batch_action_model.resource:
example_return_value = xform_name(batch_action_model.resource.type)
example_resource_name = xform_name(resource_name)
if service_model.service_name == resource_name:
example_resource_name = resource_name
example_prefix = '{} = {}.{}.{}'.format(
example_return_value,
example_resource_name,
collection_model.name,
batch_action_model.name,
)
document_model_driven_resource_method(
section=section,
method_name=batch_action_model.name,
operation_model=operation_model,
event_emitter=event_emitter,
method_description=operation_model.documentation,
example_prefix=example_prefix,
exclude_input=ignore_params,
resource_action_model=batch_action_model,
include_signature=include_signature,
)
def document_resource_waiter(
section,
resource_name,
event_emitter,
service_model,
resource_waiter_model,
service_waiter_model,
include_signature=True,
):
waiter_model = service_waiter_model.get_waiter(
resource_waiter_model.waiter_name
)
operation_model = service_model.operation_model(waiter_model.operation)
ignore_params = get_resource_ignore_params(resource_waiter_model.params)
service_module_name = get_service_module_name(service_model)
description = (
'Waits until this {} is {}. This method calls '
':py:meth:`{}.Waiter.{}.wait` which polls. '
':py:meth:`{}.Client.{}` every {} seconds until '
'a successful state is reached. An error is returned '
'after {} failed checks.'.format(
resource_name,
' '.join(resource_waiter_model.name.split('_')[2:]),
service_module_name,
xform_name(resource_waiter_model.waiter_name),
service_module_name,
xform_name(waiter_model.operation),
waiter_model.delay,
waiter_model.max_attempts,
)
)
example_prefix = '{}.{}'.format(
xform_name(resource_name), resource_waiter_model.name
)
document_model_driven_method(
section=section,
method_name=resource_waiter_model.name,
operation_model=operation_model,
event_emitter=event_emitter,
example_prefix=example_prefix,
method_description=description,
exclude_input=ignore_params,
include_signature=include_signature,
)
if 'return' in section.available_sections:
# Waiters do not return anything so we should remove
# any sections that may document the underlying return
# value of the client method.
return_section = section.get_section('return')
return_section.clear_text()
return_section.remove_all_sections()
return_section.write(':returns: None')
def get_attributes(self, shape):
"""
Get a dictionary of attribute names to original name and shape
models that represent the attributes of this resource. Looks
like the following:
{
'some_name': ('SomeName', <Shape...>)
}
:type shape: ibm_botocore.model.Shape
:param shape: The underlying shape for this resource.
:rtype: dict
:return: Mapping of resource attributes.
"""
attributes = {}
identifier_names = [i.name for i in self.identifiers]
for name, member in shape.members.items():
snake_cased = xform_name(name)
if snake_cased in identifier_names:
# Skip identifiers, these are set through other means
continue
snake_cased = self._get_name('attribute',
snake_cased,
snake_case=False)
attributes[snake_cased] = (name, member)
return attributes
def __init__(self, model, parent, handler, **kwargs):
self._model = model
self._parent = parent
self._py_operation_name = xform_name(
model.request.operation)
self._handler = handler
self._params = copy.deepcopy(kwargs)
def __call__(self, parent, *args, **kwargs):
"""
Perform the action's request operation after building operation
parameters and build any defined resources from the response.
:type parent: :py:class:`~ibm_boto3.resources.base.ServiceResource`
:param parent: The resource instance to which this action is attached.
:rtype: dict or ServiceResource or list(ServiceResource)
:return: The response, either as a raw dict or resource instance(s).
"""
operation_name = xform_name(self._action_model.request.operation)
# First, build predefined params and then update with the
# user-supplied kwargs, which allows overriding the pre-built
# params if needed.
params = create_request_parameters(parent, self._action_model.request)
params.update(kwargs)
logger.debug(
'Calling %s:%s with %r',
parent.meta.service_name,
operation_name,
params,
)
response = getattr(parent.meta.client, operation_name)(*args, **params)
logger.debug('Response: %r', response)
return self._response_handler(parent, params, response)
def __call__(self, parent, *args, **kwargs):
"""
Perform the wait operation after building operation
parameters.
:type parent: :py:class:`~ibm_boto3.resources.base.ServiceResource`
:param parent: The resource instance to which this action is attached.
"""
client_waiter_name = xform_name(self._waiter_model.waiter_name)
# First, build predefined params and then update with the
# user-supplied kwargs, which allows overriding the pre-built
# params if needed.
params = create_request_parameters(parent, self._waiter_model)
params.update(kwargs)
logger.debug(
'Calling %s:%s with %r',
parent.meta.service_name,
self._waiter_resource_name,
params,
)
client = parent.meta.client
waiter = client.get_waiter(client_waiter_name)
response = waiter.wait(**params)
logger.debug('Response: %r', response)
def _create_methods(self, service_model):
op_dict = {}
for operation_name in service_model.operation_names:
py_operation_name = xform_name(operation_name)
op_dict[py_operation_name] = self._create_api_method(
py_operation_name, operation_name, service_model)
return op_dict
def test_json_errors_parsing():
# The outputs/ directory has sample output responses
# For each file in outputs/ there's a corresponding file
# in expected/ that has the expected parsed response.
base_dir = os.path.join(os.path.dirname(__file__), 'json')
json_responses_dir = os.path.join(base_dir, 'errors')
expected_parsed_dir = os.path.join(base_dir, 'expected')
session = ibm_botocore.session.get_session()
for json_response_file in os.listdir(json_responses_dir):
# Files look like: 'datapipeline-create-pipeline.json'
service_name, operation_name = os.path.splitext(
json_response_file)[0].split('-', 1)
expected_parsed_response = os.path.join(expected_parsed_dir,
json_response_file)
raw_response_file = os.path.join(json_responses_dir,
json_response_file)
with open(expected_parsed_response) as f:
expected = json.load(f)
service_model = session.get_service_model(service_name)
operation_names = service_model.operation_names
operation_model = None
for op_name in operation_names:
if xform_name(op_name) == operation_name.replace('-', '_'):
operation_model = service_model.operation_model(op_name)
with open(raw_response_file, 'rb') as f:
raw_response_body = f.read()
yield _test_parsed_response, raw_response_file, \
raw_response_body, operation_model, expected
def _add_paginator(self, section, paginator_name):
section = section.add_new_section(paginator_name)
# Docment the paginator class
section.style.start_sphinx_py_class(
class_name='%s.Paginator.%s' % (
self._client.__class__.__name__, paginator_name))
section.style.start_codeblock()
section.style.new_line()
# Document how to instantiate the paginator.
section.write(
'paginator = client.get_paginator(\'%s\')' % xform_name(
paginator_name)
)
section.style.end_codeblock()
section.style.new_line()
# Get the pagination model for the particular paginator.
paginator_config = self._service_paginator_model.get_paginator(
paginator_name)
document_paginate_method(
section=section,
paginator_name=paginator_name,
event_emitter=self._client.meta.events,
service_model=self._client.meta.service_model,
paginator_config=paginator_config
)
def _load_name_with_category(self, names, name, category, snake_case=True):
"""
Load a name with a given category, possibly renaming it
if that name is already in use. The name will be stored
in ``names`` and possibly be set up in ``self._renamed``.
:type names: set
:param names: Existing names (Python attributes, properties, or
methods) on the resource.
:type name: string
:param name: The original name of the value.
:type category: string
:param category: The value type, such as 'identifier' or 'action'
:type snake_case: bool
:param snake_case: True (default) if the name should be snake cased.
"""
if snake_case:
name = xform_name(name)
if name in names:
logger.debug(f'Renaming {self.name} {category} {name}')
self._renamed[(category, name)] = name + '_' + category
name += '_' + category
if name in names:
# This isn't good, let's raise instead of trying to keep
# renaming this value.
raise ValueError('Problem renaming {} {} to {}!'.format(
self.name, category, name))
names.add(name)
def document_action(
section,
resource_name,
event_emitter,
action_model,
service_model,
include_signature=True,
):
"""Documents a resource action
:param section: The section to write to
:param resource_name: The name of the resource
:param event_emitter: The event emitter to use to emit events
:param action_model: The model of the action
:param service_model: The model of the service
:param include_signature: Whether or not to include the signature.
It is useful for generating docstrings.
"""
operation_model = service_model.operation_model(
action_model.request.operation)
ignore_params = get_resource_ignore_params(action_model.request.params)
example_return_value = 'response'
if action_model.resource:
example_return_value = xform_name(action_model.resource.type)
example_resource_name = xform_name(resource_name)
if service_model.service_name == resource_name:
example_resource_name = resource_name
example_prefix = '{} = {}.{}'.format(example_return_value,
example_resource_name,
action_model.name)
document_model_driven_resource_method(
section=section,
method_name=action_model.name,
operation_model=operation_model,
event_emitter=event_emitter,
method_description=operation_model.documentation,
example_prefix=example_prefix,
exclude_input=ignore_params,
resource_action_model=action_model,
include_signature=include_signature,
)
def document_load_reload_action(
section,
action_name,
resource_name,
event_emitter,
load_model,
service_model,
include_signature=True,
):
"""Documents the resource load action
:param section: The section to write to
:param action_name: The name of the loading action should be load or reload
:param resource_name: The name of the resource
:param event_emitter: The event emitter to use to emit events
:param load_model: The model of the load action
:param service_model: The model of the service
:param include_signature: Whether or not to include the signature.
It is useful for generating docstrings.
"""
description = (
'Calls :py:meth:`{}.Client.{}` to update the attributes of the '
'{} resource. Note that the load and reload methods are '
'the same method and can be used interchangeably.'.format(
get_service_module_name(service_model),
xform_name(load_model.request.operation),
resource_name,
))
example_resource_name = xform_name(resource_name)
if service_model.service_name == resource_name:
example_resource_name = resource_name
example_prefix = f'{example_resource_name}.{action_name}'
document_model_driven_method(
section=section,
method_name=action_name,
operation_model=OperationModel({}, service_model),
event_emitter=event_emitter,
method_description=description,
example_prefix=example_prefix,
include_signature=include_signature,
)
def _create_name_mapping(self, service_model):
# py_name -> OperationName, for every operation available
# for a service.
mapping = {}
for operation_name in service_model.operation_names:
py_operation_name = xform_name(operation_name)
mapping[py_operation_name] = operation_name
return mapping
def test_can_make_request_and_understand_errors_with_client():
session = ibm_botocore.session.get_session()
for service_name in _list_services(ERROR_TESTS):
client = _get_client(session, service_name)
for operation_name in ERROR_TESTS[service_name]:
kwargs = ERROR_TESTS[service_name][operation_name]
method_name = xform_name(operation_name)
yield _make_error_client_call, client, method_name, kwargs
def build_identifiers(identifiers, parent, params=None, raw_response=None):
"""
Builds a mapping of identifier names to values based on the
identifier source location, type, and target. Identifier
values may be scalars or lists depending on the source type
and location.
:type identifiers: list
:param identifiers: List of :py:class:`~boto3.resources.model.Parameter`
definitions
:type parent: ServiceResource
:param parent: The resource instance to which this action is attached.
:type params: dict
:param params: Request parameters sent to the service.
:type raw_response: dict
:param raw_response: Low-level operation response.
:rtype: list
:return: An ordered list of ``(name, value)`` identifier tuples.
"""
results = []
for identifier in identifiers:
source = identifier.source
target = identifier.target
if source == 'response':
value = jmespath.search(identifier.path, raw_response)
elif source == 'requestParameter':
value = jmespath.search(identifier.path, params)
elif source == 'identifier':
value = getattr(parent, xform_name(identifier.name))
elif source == 'data':
# If this is a data member then it may incur a load
# action before returning the value.
value = get_data_member(parent, identifier.path)
elif source == 'input':
# This value is set by the user, so ignore it here
continue
else:
raise NotImplementedError(
'Unsupported source type: {0}'.format(source))
results.append((xform_name(target), value))
return results
def waiter_names(self):
"""Returns a list of all available waiters."""
config = self._get_waiter_config()
if not config:
return []
model = waiter.WaiterModel(config)
# Waiter configs is a dict, we just want the waiter names
# which are the keys in the dict.
return [xform_name(name) for name in model.waiter_names]
def _make_api_call(session, api_call):
client = session.create_client(api_call['serviceId'].lower().replace(
' ', ''))
operation_name = api_call['operationName']
client_method = getattr(client, xform_name(operation_name))
with _stubbed_http_layer(client, api_call['attemptResponses']):
try:
client_method(**api_call['params'])
except EXPECTED_EXCEPTIONS_THROWN:
pass
def test_can_make_request_with_client():
# Same as test_can_make_request, but with Client objects
# instead of service/operations.
session = ibm_botocore.session.get_session()
for service_name in _list_services(SMOKE_TESTS):
client = _get_client(session, service_name)
for operation_name in SMOKE_TESTS[service_name]:
kwargs = SMOKE_TESTS[service_name][operation_name]
method_name = xform_name(operation_name)
yield _make_client_call, client, method_name, kwargs
def __call__(self, parent, *args, **kwargs):
"""
Perform the batch action's operation on every page of results
from the collection.
:type parent:
:py:class:`~ibm_boto3.resources.collection.ResourceCollection`
:param parent: The collection iterator to which this action
is attached.
:rtype: list(dict)
:return: A list of low-level response dicts from each call.
"""
service_name = None
client = None
responses = []
operation_name = xform_name(self._action_model.request.operation)
# Unlike the simple action above, a batch action must operate
# on batches (or pages) of items. So we get each page, construct
# the necessary parameters and call the batch operation.
for page in parent.pages():
params = {}
for index, resource in enumerate(page):
# There is no public interface to get a service name
# or low-level client from a collection, so we get
# these from the first resource in the collection.
if service_name is None:
service_name = resource.meta.service_name
if client is None:
client = resource.meta.client
create_request_parameters(
resource,
self._action_model.request,
params=params,
index=index,
)
if not params:
# There are no items, no need to make a call.
break
params.update(kwargs)
logger.debug('Calling %s:%s with %r', service_name, operation_name,
params)
response = getattr(client, operation_name)(*args, **params)
logger.debug('Response: %r', response)
responses.append(self._response_handler(parent, params, response))
return responses
def _load_batch_actions(self, attrs, resource_name, collection_model,
service_model, event_emitter):
"""
Batch actions on the collection become methods on both
the collection manager and iterators.
"""
for action_model in collection_model.batch_actions:
snake_cased = xform_name(action_model.name)
attrs[snake_cased] = self._create_batch_action(
resource_name, snake_cased, action_model, collection_model,
service_model, event_emitter)
def test_can_make_request_with_client(ibm_botocore_session, service_name,
operation_name, kwargs):
# Same as test_can_make_request, but with Client objects
# instead of service/operations.
client = _get_client(ibm_botocore_session, service_name)
method = getattr(client, xform_name(operation_name))
with warnings.catch_warnings(record=True) as caught_warnings:
response = method(**kwargs)
err_msg = f"Warnings were emitted during smoke test: {caught_warnings}"
assert len(caught_warnings) == 0, err_msg
assert 'Errors' not in response
def test_public_apis_will_not_be_signed():
session = Session()
# Mimic the scenario that user does not have aws credentials setup
session.get_credentials = mock.Mock(return_value=None)
for service_name in PUBLIC_API_TESTS:
client = session.create_client(service_name, REGIONS[service_name])
for operation_name in PUBLIC_API_TESTS[service_name]:
kwargs = PUBLIC_API_TESTS[service_name][operation_name]
method = getattr(client, xform_name(operation_name))
yield (_test_public_apis_will_not_be_signed, method, kwargs)
def test_all_streaming_body_are_properly_documented(self):
for service in self._session.get_available_services():
client = self._session.create_client(service,
region_name='us-east-1',
aws_access_key_id='foo',
aws_secret_access_key='bar')
service_model = client.meta.service_model
for operation in service_model.operation_names:
operation_model = service_model.operation_model(operation)
if operation_model.has_streaming_output:
self.assert_streaming_body_is_properly_documented(
service, xform_name(operation))
def _make_client_call_with_errors(client, operation_name, kwargs):
operation = getattr(client, xform_name(operation_name))
exception = ConnectionClosedError(endpoint_url='https://mock.eror')
with ClientHTTPStubber(client, strict=False) as http_stubber:
http_stubber.responses.append(exception)
try:
response = operation(**kwargs)
except ClientError as e:
assert False, ('Request was not retried properly, '
'received error:\n%s' % pformat(e))
# Ensure we used the stubber as we're not using it in strict mode
assert len(http_stubber.responses) == 0, 'Stubber was not used!'
def get_waiter(self, waiter_name):
config = self._get_waiter_config()
if not config:
raise ValueError("Waiter does not exist: %s" % waiter_name)
model = waiter.WaiterModel(config)
mapping = {}
for name in model.waiter_names:
mapping[xform_name(name)] = name
if waiter_name not in mapping:
raise ValueError("Waiter does not exist: %s" % waiter_name)
return waiter.create_waiter_with_client(mapping[waiter_name], model,
self)
def _add_example(self, section, identifier_names):
section.style.start_codeblock()
section.style.new_line()
section.write('import ibm_boto3')
section.style.new_line()
section.style.new_line()
section.write('%s = ibm_boto3.resource(\'%s\')' %
(self._service_name, self._service_name))
section.style.new_line()
example_values = get_identifier_values_for_example(identifier_names)
section.write('%s = %s.%s(%s)' %
(xform_name(self._resource_name), self._service_name,
self._resource_name, example_values))
section.style.end_codeblock()
def _get_operation_model(service_model, filename):
dirname, filename = os.path.split(filename)
basename = os.path.splitext(filename)[0]
sn, opname = basename.split('-', 1)
# In order to have multiple tests for the same
# operation a '#' char is used to separate
# operation names from some other suffix so that
# the tests have a different filename, e.g
# my-operation#1.xml, my-operation#2.xml.
opname = opname.split('#')[0]
operation_names = service_model.operation_names
for operation_name in operation_names:
if xform_name(operation_name) == opname.replace('-', '_'):
return service_model.operation_model(operation_name)
def _assert_has_waiter_documentation(generated_docs, service_name, client,
waiter_model):
ref_lines = ['=======', 'Waiters', '=======', 'The available waiters are:']
for waiter_name in waiter_model.waiter_names:
ref_lines.append('* :py:class:`{}.Waiter.{}`'.format(
client.__class__.__name__, waiter_name))
for waiter_name in waiter_model.waiter_names:
ref_lines.append('.. py:class:: {}.Waiter.{}'.format(
client.__class__.__name__, waiter_name))
ref_lines.append(' waiter = client.get_waiter(\'%s\')' %
xform_name(waiter_name))
ref_lines.append(' .. py:method:: wait(')
_assert_contains_lines_in_order(ref_lines, generated_docs)
def create_request_parameters(parent, request_model, params=None, index=None):
"""
Handle request parameters that can be filled in from identifiers,
resource data members or constants.
By passing ``params``, you can invoke this method multiple times and
build up a parameter dict over time, which is particularly useful
for reverse JMESPath expressions that append to lists.
:type parent: ServiceResource
:param parent: The resource instance to which this action is attached.
:type request_model: :py:class:`~ibm_boto3.resources.model.Request`
:param request_model: The action request model.
:type params: dict
:param params: If set, then add to this existing dict. It is both
edited in-place and returned.
:type index: int
:param index: The position of an item within a list
:rtype: dict
:return: Pre-filled parameters to be sent to the request operation.
"""
if params is None:
params = {}
for param in request_model.params:
source = param.source
target = param.target
if source == 'identifier':
# Resource identifier, e.g. queue.url
value = getattr(parent, xform_name(param.name))
elif source == 'data':
# If this is a data member then it may incur a load
# action before returning the value.
value = get_data_member(parent, param.path)
elif source in ['string', 'integer', 'boolean']:
# These are hard-coded values in the definition
value = param.value
elif source == 'input':
# This is provided by the user, so ignore it here
continue
else:
raise NotImplementedError(
'Unsupported source type: {0}'.format(source))
build_param_structure(params, target, value, index)
return params
def _make_client_call_with_errors(client, operation_name, kwargs):
operation = getattr(client, xform_name(operation_name))
original_send = adapters.HTTPAdapter.send
def mock_http_adapter_send(self, *args, **kwargs):
if not getattr(self, '_integ_test_error_raised', False):
self._integ_test_error_raised = True
raise ConnectionError("Simulated ConnectionError raised.")
else:
return original_send(self, *args, **kwargs)
with mock.patch('ibm_botocore.vendored.requests.adapters.HTTPAdapter.send',
mock_http_adapter_send):
try:
response = operation(**kwargs)
except ClientError as e:
assert False, ('Request was not retried properly, '
'received error:\n%s' % pformat(e))
