def build_full_result(self): complete_result = {} # Prepopulate the result keys with an empty list. for result_expression in self.result_keys: set_value_from_jmespath(complete_result, result_expression.expression, []) for _, page in self: # We're incrementally building the full response page # by page. For each page in the response we need to # inject the necessary components from the page # into the complete_result. for result_expression in self.result_keys: # In order to incrementally update a result key # we need to search the existing value from complete_result, # then we need to search the _current_ page for the # current result key value. Then we append the current # value onto the existing value, and re-set that value # as the new value. existing_value = result_expression.search(complete_result) result_value = result_expression.search(page) if result_value is not None: existing_value.extend(result_value) merge_dicts(complete_result, self.non_aggregate_part) if self.resume_token is not None: complete_result['NextToken'] = self.resume_token return complete_result
def _record_non_aggregate_key_values(self, response): non_aggregate_keys = {} for expression in self._non_aggregate_key_exprs: result = expression.search(response) set_value_from_jmespath(non_aggregate_keys, expression.expression, result) self._non_aggregate_part = non_aggregate_keys
def _truncate_response(self, parsed, primary_result_key, truncate_amount, starting_truncation, next_token): original = primary_result_key.search(parsed) if original is None: original = [] amount_to_keep = len(original) - truncate_amount truncated = original[:amount_to_keep] set_value_from_jmespath( parsed, primary_result_key.expression, truncated ) # The issue here is that even though we know how much we've truncated # we need to account for this globally including any starting # left truncation. For example: # Raw response: [0,1,2,3] # Starting index: 1 # Max items: 1 # Starting left truncation: [1, 2, 3] # End right truncation for max items: [1] # However, even though we only kept 1, this is post # left truncation so the next starting index should be 2, not 1 # (left_truncation + amount_to_keep). next_token.append(str(amount_to_keep + starting_truncation)) self.resume_token = next_token
def _handle_first_request(self, parsed, primary_result_key, starting_truncation): # First we need to slice into the array and only return # the truncated amount. starting_truncation = self._parse_starting_token()[1] all_data = primary_result_key.search(parsed) set_value_from_jmespath(parsed, primary_result_key.expression, all_data[starting_truncation:]) # We also need to truncate any secondary result keys # because they were not truncated in the previous last # response. for token in self.result_keys: if token == primary_result_key: continue set_value_from_jmespath(parsed, token.expression, []) return starting_truncation
def _handle_first_request(self, parsed, primary_result_key, starting_truncation): # First we need to slice into the array and only return # the truncated amount. starting_truncation = self._parse_starting_token()[1] all_data = primary_result_key.search(parsed) set_value_from_jmespath( parsed, primary_result_key.expression, all_data[starting_truncation:] ) # We also need to truncate any secondary result keys # because they were not truncated in the previous last # response. for token in self.result_keys: if token == primary_result_key: continue set_value_from_jmespath(parsed, token.expression, []) return starting_truncation
def _truncate_response(self, parsed, primary_result_key, truncate_amount, starting_truncation, next_token): original = primary_result_key.search(parsed) if original is None: original = [] amount_to_keep = len(original) - truncate_amount truncated = original[:amount_to_keep] set_value_from_jmespath(parsed, primary_result_key.expression, truncated) # The issue here is that even though we know how much we've truncated # we need to account for this globally including any starting # left truncation. For example: # Raw response: [0,1,2,3] # Starting index: 1 # Max items: 1 # Starting left truncation: [1, 2, 3] # End right truncation for max items: [1] # However, even though we only kept 1, this is post # left truncation so the next starting index should be 2, not 1 # (left_truncation + amount_to_keep). next_token.append(str(amount_to_keep + starting_truncation)) self.resume_token = next_token