def build_retry_config(endpoint_prefix, retry_model, definitions): service_config = retry_model.get(endpoint_prefix, {}) resolve_references(service_config, definitions) # We want to merge the global defaults with the service specific # defaults, with the service specific defaults taking precedence. # So we use the global defaults as the base. final_retry_config = {'__default__': retry_model.get('__default__', {})} resolve_references(final_retry_config, definitions) # The merge the service specific config on top. merge_dicts(final_retry_config, service_config) return final_retry_config
def build_full_result(self): complete_result = {} for response in self: page = response # We want to try to catch operation object pagination # and format correctly for those. They come in the form # of a tuple of two elements: (http_response, parsed_responsed). # We want the parsed_response as that is what the page iterator # uses. We can remove it though once operation objects are removed. if isinstance(response, tuple) and len(response) == 2: page = response[1] # We're incrementally building the full response page # by page. For each page in the response we need to # inject the necessary components from the page # into the complete_result. for result_expression in self.result_keys: # In order to incrementally update a result key # we need to search the existing value from complete_result, # then we need to search the _current_ page for the # current result key value. Then we append the current # value onto the existing value, and re-set that value # as the new value. result_value = result_expression.search(page) if result_value is None: continue existing_value = result_expression.search(complete_result) if existing_value is None: # Set the initial result set_value_from_jmespath(complete_result, result_expression.expression, result_value) continue # Now both result_value and existing_value contain something if isinstance(result_value, list): existing_value.extend(result_value) elif isinstance(result_value, (int, float, six.string_types)): # Modify the existing result with the sum or concatenation set_value_from_jmespath(complete_result, result_expression.expression, existing_value + result_value) merge_dicts(complete_result, self.non_aggregate_part) if self.resume_token is not None: complete_result['NextToken'] = self.resume_token return complete_result
def build_retry_config(endpoint_prefix, retry_model, definitions, client_retry_config=None): service_config = retry_model.get(endpoint_prefix, {}) resolve_references(service_config, definitions) # We want to merge the global defaults with the service specific # defaults, with the service specific defaults taking precedence. # So we use the global defaults as the base. # # A deepcopy is done on the retry defaults because it ensures the # retry model has no chance of getting mutated when the service specific # configuration or client retry config is merged in. final_retry_config = { '__default__': copy.deepcopy(retry_model.get('__default__', {})) } resolve_references(final_retry_config, definitions) # The merge the service specific config on top. merge_dicts(final_retry_config, service_config) if client_retry_config is not None: _merge_client_retry_config(final_retry_config, client_retry_config) return final_retry_config
def _parse_error_from_body(self, response): xml_contents = response['body'] root = self._parse_xml_string_to_dom(xml_contents) parsed = self._build_name_to_xml_node(root) self._replace_nodes(parsed) if root.tag == 'Error': # This is an S3 error response. First we'll populate the # response metadata. metadata = self._populate_response_metadata(response) # The RequestId and the HostId are already in the # ResponseMetadata, but are also duplicated in the XML # body. We don't need these values in both places, # we'll just remove them from the parsed XML body. parsed.pop('RequestId', '') parsed.pop('HostId', '') return {'Error': parsed, 'ResponseMetadata': metadata} elif 'RequestId' in parsed: # Other rest-xml serivces: parsed['ResponseMetadata'] = {'RequestId': parsed.pop('RequestId')} default = {'Error': {'Message': '', 'Code': ''}} merge_dicts(default, parsed) return default
def _clone(self, **kwargs): """ Create a clone of this collection. This is used by the methods below to provide a chainable interface that returns copies rather than the original. This allows things like: >>> base = collection.filter(Param1=1) >>> query1 = base.filter(Param2=2) >>> query2 = base.filter(Param3=3) >>> query1.params {'Param1': 1, 'Param2': 2} >>> query2.params {'Param1': 1, 'Param3': 3} :rtype: :py:class:`ResourceCollection` :return: A clone of this resource collection """ params = copy.deepcopy(self._params) merge_dicts(params, kwargs, append_lists=True) clone = self.__class__(self._model, self._parent, self._handler, **params) return clone
def pages(self): """ A generator which yields pages of resource instances after doing the appropriate service operation calls and handling any pagination on your behalf. Non-paginated calls will return a single page of items. Page size, item limit, and filter parameters are applied if they have previously been set. >>> bucket = s3.Bucket('boto3') >>> for page in bucket.objects.pages(): ... for obj in page: ... print(obj.key) 'key1' 'key2' :rtype: list(:py:class:`~boto3.resources.base.ServiceResource`) :return: List of resource instances """ client = self._parent.meta.client cleaned_params = self._params.copy() limit = cleaned_params.pop('limit', None) page_size = cleaned_params.pop('page_size', None) params = create_request_parameters(self._parent, self._model.request) merge_dicts(params, cleaned_params, append_lists=True) # Is this a paginated operation? If so, we need to get an # iterator for the various pages. If not, then we simply # call the operation and return the result as a single # page in a list. For non-paginated results, we just ignore # the page size parameter. if client.can_paginate(self._py_operation_name): logger.info('Calling paginated %s:%s with %r', self._parent.meta.service_name, self._py_operation_name, params) paginator = client.get_paginator(self._py_operation_name) pages = paginator.paginate(PaginationConfig={ 'MaxItems': limit, 'PageSize': page_size }, **params) else: logger.info('Calling %s:%s with %r', self._parent.meta.service_name, self._py_operation_name, params) pages = [getattr(client, self._py_operation_name)(**params)] # Now that we have a page iterator or single page of results # we start processing and yielding individual items. count = 0 for page in pages: page_items = [] for item in self._handler(self._parent, params, page): page_items.append(item) # If the limit is set and has been reached, then # we stop processing items here. count += 1 if limit is not None and count >= limit: break yield page_items # Stop reading pages if we've reached out limit if limit is not None and count >= limit: break