def document_batch_action( section, resource_name, event_emitter, batch_action_model, service_model, collection_model, include_signature=True, ): """Documents a collection's batch action :param section: The section to write to :param resource_name: The name of the resource :param action_name: The name of collection action. Currently only can be all, filter, limit, or page_size :param event_emitter: The event emitter to use to emit events :param batch_action_model: The model of the batch action :param collection_model: The model of the collection :param service_model: The model of the service :param include_signature: Whether or not to include the signature. It is useful for generating docstrings. """ operation_model = service_model.operation_model( batch_action_model.request.operation ) ignore_params = get_resource_ignore_params( batch_action_model.request.params ) example_return_value = 'response' if batch_action_model.resource: example_return_value = xform_name(batch_action_model.resource.type) example_resource_name = xform_name(resource_name) if service_model.service_name == resource_name: example_resource_name = resource_name example_prefix = '{} = {}.{}.{}'.format( example_return_value, example_resource_name, collection_model.name, batch_action_model.name, ) document_model_driven_resource_method( section=section, method_name=batch_action_model.name, operation_model=operation_model, event_emitter=event_emitter, method_description=operation_model.documentation, example_prefix=example_prefix, exclude_input=ignore_params, resource_action_model=batch_action_model, include_signature=include_signature, )
def document_resource_waiter( section, resource_name, event_emitter, service_model, resource_waiter_model, service_waiter_model, include_signature=True, ): waiter_model = service_waiter_model.get_waiter( resource_waiter_model.waiter_name ) operation_model = service_model.operation_model(waiter_model.operation) ignore_params = get_resource_ignore_params(resource_waiter_model.params) service_module_name = get_service_module_name(service_model) description = ( 'Waits until this {} is {}. This method calls ' ':py:meth:`{}.Waiter.{}.wait` which polls. ' ':py:meth:`{}.Client.{}` every {} seconds until ' 'a successful state is reached. An error is returned ' 'after {} failed checks.'.format( resource_name, ' '.join(resource_waiter_model.name.split('_')[2:]), service_module_name, xform_name(resource_waiter_model.waiter_name), service_module_name, xform_name(waiter_model.operation), waiter_model.delay, waiter_model.max_attempts, ) ) example_prefix = '{}.{}'.format( xform_name(resource_name), resource_waiter_model.name ) document_model_driven_method( section=section, method_name=resource_waiter_model.name, operation_model=operation_model, event_emitter=event_emitter, example_prefix=example_prefix, method_description=description, exclude_input=ignore_params, include_signature=include_signature, ) if 'return' in section.available_sections: # Waiters do not return anything so we should remove # any sections that may document the underlying return # value of the client method. return_section = section.get_section('return') return_section.clear_text() return_section.remove_all_sections() return_section.write(':returns: None')
def get_attributes(self, shape): """ Get a dictionary of attribute names to original name and shape models that represent the attributes of this resource. Looks like the following: { 'some_name': ('SomeName', <Shape...>) } :type shape: ibm_botocore.model.Shape :param shape: The underlying shape for this resource. :rtype: dict :return: Mapping of resource attributes. """ attributes = {} identifier_names = [i.name for i in self.identifiers] for name, member in shape.members.items(): snake_cased = xform_name(name) if snake_cased in identifier_names: # Skip identifiers, these are set through other means continue snake_cased = self._get_name('attribute', snake_cased, snake_case=False) attributes[snake_cased] = (name, member) return attributes
def __init__(self, model, parent, handler, **kwargs): self._model = model self._parent = parent self._py_operation_name = xform_name( model.request.operation) self._handler = handler self._params = copy.deepcopy(kwargs)
def __call__(self, parent, *args, **kwargs): """ Perform the action's request operation after building operation parameters and build any defined resources from the response. :type parent: :py:class:`~ibm_boto3.resources.base.ServiceResource` :param parent: The resource instance to which this action is attached. :rtype: dict or ServiceResource or list(ServiceResource) :return: The response, either as a raw dict or resource instance(s). """ operation_name = xform_name(self._action_model.request.operation) # First, build predefined params and then update with the # user-supplied kwargs, which allows overriding the pre-built # params if needed. params = create_request_parameters(parent, self._action_model.request) params.update(kwargs) logger.debug( 'Calling %s:%s with %r', parent.meta.service_name, operation_name, params, ) response = getattr(parent.meta.client, operation_name)(*args, **params) logger.debug('Response: %r', response) return self._response_handler(parent, params, response)
def __call__(self, parent, *args, **kwargs): """ Perform the wait operation after building operation parameters. :type parent: :py:class:`~ibm_boto3.resources.base.ServiceResource` :param parent: The resource instance to which this action is attached. """ client_waiter_name = xform_name(self._waiter_model.waiter_name) # First, build predefined params and then update with the # user-supplied kwargs, which allows overriding the pre-built # params if needed. params = create_request_parameters(parent, self._waiter_model) params.update(kwargs) logger.debug( 'Calling %s:%s with %r', parent.meta.service_name, self._waiter_resource_name, params, ) client = parent.meta.client waiter = client.get_waiter(client_waiter_name) response = waiter.wait(**params) logger.debug('Response: %r', response)
def _create_methods(self, service_model): op_dict = {} for operation_name in service_model.operation_names: py_operation_name = xform_name(operation_name) op_dict[py_operation_name] = self._create_api_method( py_operation_name, operation_name, service_model) return op_dict
def test_json_errors_parsing(): # The outputs/ directory has sample output responses # For each file in outputs/ there's a corresponding file # in expected/ that has the expected parsed response. base_dir = os.path.join(os.path.dirname(__file__), 'json') json_responses_dir = os.path.join(base_dir, 'errors') expected_parsed_dir = os.path.join(base_dir, 'expected') session = ibm_botocore.session.get_session() for json_response_file in os.listdir(json_responses_dir): # Files look like: 'datapipeline-create-pipeline.json' service_name, operation_name = os.path.splitext( json_response_file)[0].split('-', 1) expected_parsed_response = os.path.join(expected_parsed_dir, json_response_file) raw_response_file = os.path.join(json_responses_dir, json_response_file) with open(expected_parsed_response) as f: expected = json.load(f) service_model = session.get_service_model(service_name) operation_names = service_model.operation_names operation_model = None for op_name in operation_names: if xform_name(op_name) == operation_name.replace('-', '_'): operation_model = service_model.operation_model(op_name) with open(raw_response_file, 'rb') as f: raw_response_body = f.read() yield _test_parsed_response, raw_response_file, \ raw_response_body, operation_model, expected
def _add_paginator(self, section, paginator_name): section = section.add_new_section(paginator_name) # Docment the paginator class section.style.start_sphinx_py_class( class_name='%s.Paginator.%s' % ( self._client.__class__.__name__, paginator_name)) section.style.start_codeblock() section.style.new_line() # Document how to instantiate the paginator. section.write( 'paginator = client.get_paginator(\'%s\')' % xform_name( paginator_name) ) section.style.end_codeblock() section.style.new_line() # Get the pagination model for the particular paginator. paginator_config = self._service_paginator_model.get_paginator( paginator_name) document_paginate_method( section=section, paginator_name=paginator_name, event_emitter=self._client.meta.events, service_model=self._client.meta.service_model, paginator_config=paginator_config )
def _load_name_with_category(self, names, name, category, snake_case=True): """ Load a name with a given category, possibly renaming it if that name is already in use. The name will be stored in ``names`` and possibly be set up in ``self._renamed``. :type names: set :param names: Existing names (Python attributes, properties, or methods) on the resource. :type name: string :param name: The original name of the value. :type category: string :param category: The value type, such as 'identifier' or 'action' :type snake_case: bool :param snake_case: True (default) if the name should be snake cased. """ if snake_case: name = xform_name(name) if name in names: logger.debug(f'Renaming {self.name} {category} {name}') self._renamed[(category, name)] = name + '_' + category name += '_' + category if name in names: # This isn't good, let's raise instead of trying to keep # renaming this value. raise ValueError('Problem renaming {} {} to {}!'.format( self.name, category, name)) names.add(name)
def document_action( section, resource_name, event_emitter, action_model, service_model, include_signature=True, ): """Documents a resource action :param section: The section to write to :param resource_name: The name of the resource :param event_emitter: The event emitter to use to emit events :param action_model: The model of the action :param service_model: The model of the service :param include_signature: Whether or not to include the signature. It is useful for generating docstrings. """ operation_model = service_model.operation_model( action_model.request.operation) ignore_params = get_resource_ignore_params(action_model.request.params) example_return_value = 'response' if action_model.resource: example_return_value = xform_name(action_model.resource.type) example_resource_name = xform_name(resource_name) if service_model.service_name == resource_name: example_resource_name = resource_name example_prefix = '{} = {}.{}'.format(example_return_value, example_resource_name, action_model.name) document_model_driven_resource_method( section=section, method_name=action_model.name, operation_model=operation_model, event_emitter=event_emitter, method_description=operation_model.documentation, example_prefix=example_prefix, exclude_input=ignore_params, resource_action_model=action_model, include_signature=include_signature, )
def document_load_reload_action( section, action_name, resource_name, event_emitter, load_model, service_model, include_signature=True, ): """Documents the resource load action :param section: The section to write to :param action_name: The name of the loading action should be load or reload :param resource_name: The name of the resource :param event_emitter: The event emitter to use to emit events :param load_model: The model of the load action :param service_model: The model of the service :param include_signature: Whether or not to include the signature. It is useful for generating docstrings. """ description = ( 'Calls :py:meth:`{}.Client.{}` to update the attributes of the ' '{} resource. Note that the load and reload methods are ' 'the same method and can be used interchangeably.'.format( get_service_module_name(service_model), xform_name(load_model.request.operation), resource_name, )) example_resource_name = xform_name(resource_name) if service_model.service_name == resource_name: example_resource_name = resource_name example_prefix = f'{example_resource_name}.{action_name}' document_model_driven_method( section=section, method_name=action_name, operation_model=OperationModel({}, service_model), event_emitter=event_emitter, method_description=description, example_prefix=example_prefix, include_signature=include_signature, )
def _create_name_mapping(self, service_model): # py_name -> OperationName, for every operation available # for a service. mapping = {} for operation_name in service_model.operation_names: py_operation_name = xform_name(operation_name) mapping[py_operation_name] = operation_name return mapping
def test_can_make_request_and_understand_errors_with_client(): session = ibm_botocore.session.get_session() for service_name in _list_services(ERROR_TESTS): client = _get_client(session, service_name) for operation_name in ERROR_TESTS[service_name]: kwargs = ERROR_TESTS[service_name][operation_name] method_name = xform_name(operation_name) yield _make_error_client_call, client, method_name, kwargs
def build_identifiers(identifiers, parent, params=None, raw_response=None): """ Builds a mapping of identifier names to values based on the identifier source location, type, and target. Identifier values may be scalars or lists depending on the source type and location. :type identifiers: list :param identifiers: List of :py:class:`~boto3.resources.model.Parameter` definitions :type parent: ServiceResource :param parent: The resource instance to which this action is attached. :type params: dict :param params: Request parameters sent to the service. :type raw_response: dict :param raw_response: Low-level operation response. :rtype: list :return: An ordered list of ``(name, value)`` identifier tuples. """ results = [] for identifier in identifiers: source = identifier.source target = identifier.target if source == 'response': value = jmespath.search(identifier.path, raw_response) elif source == 'requestParameter': value = jmespath.search(identifier.path, params) elif source == 'identifier': value = getattr(parent, xform_name(identifier.name)) elif source == 'data': # If this is a data member then it may incur a load # action before returning the value. value = get_data_member(parent, identifier.path) elif source == 'input': # This value is set by the user, so ignore it here continue else: raise NotImplementedError( 'Unsupported source type: {0}'.format(source)) results.append((xform_name(target), value)) return results
def waiter_names(self): """Returns a list of all available waiters.""" config = self._get_waiter_config() if not config: return [] model = waiter.WaiterModel(config) # Waiter configs is a dict, we just want the waiter names # which are the keys in the dict. return [xform_name(name) for name in model.waiter_names]
def _make_api_call(session, api_call): client = session.create_client(api_call['serviceId'].lower().replace( ' ', '')) operation_name = api_call['operationName'] client_method = getattr(client, xform_name(operation_name)) with _stubbed_http_layer(client, api_call['attemptResponses']): try: client_method(**api_call['params']) except EXPECTED_EXCEPTIONS_THROWN: pass
def test_can_make_request_with_client(): # Same as test_can_make_request, but with Client objects # instead of service/operations. session = ibm_botocore.session.get_session() for service_name in _list_services(SMOKE_TESTS): client = _get_client(session, service_name) for operation_name in SMOKE_TESTS[service_name]: kwargs = SMOKE_TESTS[service_name][operation_name] method_name = xform_name(operation_name) yield _make_client_call, client, method_name, kwargs
def __call__(self, parent, *args, **kwargs): """ Perform the batch action's operation on every page of results from the collection. :type parent: :py:class:`~ibm_boto3.resources.collection.ResourceCollection` :param parent: The collection iterator to which this action is attached. :rtype: list(dict) :return: A list of low-level response dicts from each call. """ service_name = None client = None responses = [] operation_name = xform_name(self._action_model.request.operation) # Unlike the simple action above, a batch action must operate # on batches (or pages) of items. So we get each page, construct # the necessary parameters and call the batch operation. for page in parent.pages(): params = {} for index, resource in enumerate(page): # There is no public interface to get a service name # or low-level client from a collection, so we get # these from the first resource in the collection. if service_name is None: service_name = resource.meta.service_name if client is None: client = resource.meta.client create_request_parameters( resource, self._action_model.request, params=params, index=index, ) if not params: # There are no items, no need to make a call. break params.update(kwargs) logger.debug('Calling %s:%s with %r', service_name, operation_name, params) response = getattr(client, operation_name)(*args, **params) logger.debug('Response: %r', response) responses.append(self._response_handler(parent, params, response)) return responses
def _load_batch_actions(self, attrs, resource_name, collection_model, service_model, event_emitter): """ Batch actions on the collection become methods on both the collection manager and iterators. """ for action_model in collection_model.batch_actions: snake_cased = xform_name(action_model.name) attrs[snake_cased] = self._create_batch_action( resource_name, snake_cased, action_model, collection_model, service_model, event_emitter)
def test_can_make_request_with_client(ibm_botocore_session, service_name, operation_name, kwargs): # Same as test_can_make_request, but with Client objects # instead of service/operations. client = _get_client(ibm_botocore_session, service_name) method = getattr(client, xform_name(operation_name)) with warnings.catch_warnings(record=True) as caught_warnings: response = method(**kwargs) err_msg = f"Warnings were emitted during smoke test: {caught_warnings}" assert len(caught_warnings) == 0, err_msg assert 'Errors' not in response
def test_public_apis_will_not_be_signed(): session = Session() # Mimic the scenario that user does not have aws credentials setup session.get_credentials = mock.Mock(return_value=None) for service_name in PUBLIC_API_TESTS: client = session.create_client(service_name, REGIONS[service_name]) for operation_name in PUBLIC_API_TESTS[service_name]: kwargs = PUBLIC_API_TESTS[service_name][operation_name] method = getattr(client, xform_name(operation_name)) yield (_test_public_apis_will_not_be_signed, method, kwargs)
def test_all_streaming_body_are_properly_documented(self): for service in self._session.get_available_services(): client = self._session.create_client(service, region_name='us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') service_model = client.meta.service_model for operation in service_model.operation_names: operation_model = service_model.operation_model(operation) if operation_model.has_streaming_output: self.assert_streaming_body_is_properly_documented( service, xform_name(operation))
def _make_client_call_with_errors(client, operation_name, kwargs): operation = getattr(client, xform_name(operation_name)) exception = ConnectionClosedError(endpoint_url='https://mock.eror') with ClientHTTPStubber(client, strict=False) as http_stubber: http_stubber.responses.append(exception) try: response = operation(**kwargs) except ClientError as e: assert False, ('Request was not retried properly, ' 'received error:\n%s' % pformat(e)) # Ensure we used the stubber as we're not using it in strict mode assert len(http_stubber.responses) == 0, 'Stubber was not used!'
def get_waiter(self, waiter_name): config = self._get_waiter_config() if not config: raise ValueError("Waiter does not exist: %s" % waiter_name) model = waiter.WaiterModel(config) mapping = {} for name in model.waiter_names: mapping[xform_name(name)] = name if waiter_name not in mapping: raise ValueError("Waiter does not exist: %s" % waiter_name) return waiter.create_waiter_with_client(mapping[waiter_name], model, self)
def _add_example(self, section, identifier_names): section.style.start_codeblock() section.style.new_line() section.write('import ibm_boto3') section.style.new_line() section.style.new_line() section.write('%s = ibm_boto3.resource(\'%s\')' % (self._service_name, self._service_name)) section.style.new_line() example_values = get_identifier_values_for_example(identifier_names) section.write('%s = %s.%s(%s)' % (xform_name(self._resource_name), self._service_name, self._resource_name, example_values)) section.style.end_codeblock()
def _get_operation_model(service_model, filename): dirname, filename = os.path.split(filename) basename = os.path.splitext(filename)[0] sn, opname = basename.split('-', 1) # In order to have multiple tests for the same # operation a '#' char is used to separate # operation names from some other suffix so that # the tests have a different filename, e.g # my-operation#1.xml, my-operation#2.xml. opname = opname.split('#')[0] operation_names = service_model.operation_names for operation_name in operation_names: if xform_name(operation_name) == opname.replace('-', '_'): return service_model.operation_model(operation_name)
def _assert_has_waiter_documentation(generated_docs, service_name, client, waiter_model): ref_lines = ['=======', 'Waiters', '=======', 'The available waiters are:'] for waiter_name in waiter_model.waiter_names: ref_lines.append('* :py:class:`{}.Waiter.{}`'.format( client.__class__.__name__, waiter_name)) for waiter_name in waiter_model.waiter_names: ref_lines.append('.. py:class:: {}.Waiter.{}'.format( client.__class__.__name__, waiter_name)) ref_lines.append(' waiter = client.get_waiter(\'%s\')' % xform_name(waiter_name)) ref_lines.append(' .. py:method:: wait(') _assert_contains_lines_in_order(ref_lines, generated_docs)
def create_request_parameters(parent, request_model, params=None, index=None): """ Handle request parameters that can be filled in from identifiers, resource data members or constants. By passing ``params``, you can invoke this method multiple times and build up a parameter dict over time, which is particularly useful for reverse JMESPath expressions that append to lists. :type parent: ServiceResource :param parent: The resource instance to which this action is attached. :type request_model: :py:class:`~ibm_boto3.resources.model.Request` :param request_model: The action request model. :type params: dict :param params: If set, then add to this existing dict. It is both edited in-place and returned. :type index: int :param index: The position of an item within a list :rtype: dict :return: Pre-filled parameters to be sent to the request operation. """ if params is None: params = {} for param in request_model.params: source = param.source target = param.target if source == 'identifier': # Resource identifier, e.g. queue.url value = getattr(parent, xform_name(param.name)) elif source == 'data': # If this is a data member then it may incur a load # action before returning the value. value = get_data_member(parent, param.path) elif source in ['string', 'integer', 'boolean']: # These are hard-coded values in the definition value = param.value elif source == 'input': # This is provided by the user, so ignore it here continue else: raise NotImplementedError( 'Unsupported source type: {0}'.format(source)) build_param_structure(params, target, value, index) return params
def _make_client_call_with_errors(client, operation_name, kwargs): operation = getattr(client, xform_name(operation_name)) original_send = adapters.HTTPAdapter.send def mock_http_adapter_send(self, *args, **kwargs): if not getattr(self, '_integ_test_error_raised', False): self._integ_test_error_raised = True raise ConnectionError("Simulated ConnectionError raised.") else: return original_send(self, *args, **kwargs) with mock.patch('ibm_botocore.vendored.requests.adapters.HTTPAdapter.send', mock_http_adapter_send): try: response = operation(**kwargs) except ClientError as e: assert False, ('Request was not retried properly, ' 'received error:\n%s' % pformat(e))