def test_max_items_as_strings(self):
# Some services (route53) model MaxItems as a string type.
# We need to be able to handle this case.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
# Note MaxItems is a string here.
paginator.paginate(PaginationConfig={
'MaxItems': '1'
}).build_full_result(),
{
'Users': ['User1'],
'NextToken': 'm1'
})
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'more_results': 'IsTruncated',
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [{
'Foo': [{
'a': 1
}, {
'b': 2
}],
'IsTruncated': True,
'NextToken': '1'
}, {
'Foo': [{
'a': 3
}, {
'b': 4
}],
'IsTruncated': True,
'NextToken': '2'
}, {
'Foo': [{
'a': 5
}],
'IsTruncated': False,
'NextToken': '3'
}]
self.method.side_effect = responses
def test_include_with_nested_result_keys(self):
self.paginate_config['result_key'] = 'Result.Key'
self.paginate_config['non_aggregate_keys'] = [
'Outer', 'Result.Inner',
]
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
self.set_responses([
# The non result keys shows hypothetical
# example. This doesn't actually happen,
# but in the case where the non result keys
# are different across pages, we use the values
# from the first page.
{'Result': {'Key': ['foo'], 'Inner': 'v1'},
'Outer': 'v2', 'NextToken': 't1'},
{'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'},
'Outer': 'v4', 'NextToken': 't2'},
{'Result': {'Key': ['qux'], 'Inner': 'v5'},
'Outer': 'v6', 'NextToken': 't3'},
])
pages = self.paginator.paginate(None)
actual = pages.build_full_result()
self.assertEqual(pages.non_aggregate_part,
{'Outer': 'v2', 'Result': {'Inner': 'v1'}})
expected = {
'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'},
'Outer': 'v2',
}
self.assertEqual(actual, expected)
def get_paginator(self, operation_name):
"""Create a paginator for an operation.
:type operation_name: string
:param operation_name: The operation name. This is the same name
as the method name on the client. For example, if the
method name is ``create_foo``, and you'd normally invoke the
operation as ``client.create_foo(**kwargs)``, if the
``create_foo`` operation can be paginated, you can use the
call ``client.get_paginator("create_foo")``.
:raise OperationNotPageableError: Raised if the operation is not
pageable. You can use the ``client.can_paginate`` method to
check if an operation is pageable.
:rtype: L{botocore.paginate.Paginator}
:return: A paginator object.
"""
if not self.can_paginate(operation_name):
raise OperationNotPageableError(operation_name=operation_name)
else:
actual_operation_name = self._PY_TO_OP_NAME[operation_name]
paginator = Paginator(
getattr(self, operation_name),
self._cache['page_config'][actual_operation_name])
return paginator
def test_resume_next_marker_mid_page(self):
# This is a simulation of picking up from the response
# from test_MaxItems_can_be_specified_truncates_response
# We got the first 4 users, when we pick up we should get
# User5 - User7.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User4", "User5", "User6"],
"Marker": "m2"
},
{
"Users": ["User7"]
},
]
self.method.side_effect = responses
starting_token = encode_token({
"Marker": "m1",
"boto_truncate_amount": 1
})
pagination_config = {'StartingToken': starting_token}
self.assertEqual(
paginator.paginate(
PaginationConfig=pagination_config).build_full_result(),
{'Users': ['User5', 'User6', 'User7']})
self.assertEqual(self.method.call_args_list,
[mock.call(Marker='m1'),
mock.call(Marker='m2')])
def test_page_size(self):
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(PaginationConfig={'PageSize': 1}):
users += page['Users']
self.assertEqual(self.method.call_args_list, [
mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)
])
def setUp(self):
self.method = mock.Mock()
# This is something like what we'd see in RDS.
self.paginate_config = {
"input_token": "Marker",
"output_token": "Marker",
"limit_key": "MaxRecords",
"result_key": "EngineDefaults.Parameters"
}
self.paginator = Paginator(self.method, self.paginate_config)
self.responses = [{
"EngineDefaults": {
"Parameters": ["One", "Two"]
},
"Marker": "m1"
}, {
"EngineDefaults": {
"Parameters": ["Three", "Four"]
},
"Marker": "m2"
}, {
"EngineDefaults": {
"Parameters": ["Five"]
}
}]
def test_next_token_with_or_expression(self):
self.pagination_config = {
'output_token': 'NextToken || NextToken2',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.pagination_config)
# Verify that despite varying between NextToken and NextToken2
# we still can extract the right next tokens.
responses = [
{
'NextToken': 'token1'
},
{
'NextToken2': 'token2'
},
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
{
'NextToken': 'token3',
'NextToken2': 'token4'
},
{
'not_next_token': 'foo'
},
]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(self.method.call_args_list, [
mock.call(),
mock.call(NextToken='token1'),
mock.call(NextToken='token2'),
mock.call(NextToken='token3')
])
def test_more_tokens_is_path_expression(self):
self.paginate_config = {
'more_results': 'Foo.IsTruncated',
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Bar',
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [
{
'Foo': {
'IsTruncated': True
},
'NextToken': 'token1'
},
{
'Foo': {
'IsTruncated': False
},
'NextToken': 'token2'
},
]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(
self.method.call_args_list,
[mock.call(), mock.call(NextToken='token1')])
def get_protections_paginator(client):
return Paginator(
client.list_protections, {
'input_token': 'NextToken',
'output_token': 'NextToken',
'result_key': 'Protections'
}, client.meta.service_model.operation_model('ListProtections'))
def test_include_with_multiple_result_keys(self):
self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2']
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation, self.paginate_config)
self.set_responses([{
'ResultKey1': ['a', 'b'],
'ResultKey2': ['u', 'v'],
'NotResultKey': 'a',
'NextToken': 'token1'
}, {
'ResultKey1': ['c', 'd'],
'ResultKey2': ['w', 'x'],
'NotResultKey': 'a',
'NextToken': 'token2'
}, {
'ResultKey1': ['e', 'f'],
'ResultKey2': ['y', 'z'],
'NotResultKey': 'a',
}])
pages = self.paginator.paginate(None)
actual = pages.build_full_result()
expected = {
'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'],
'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'],
'NotResultKey': 'a',
}
self.assertEqual(actual, expected)
def test_build_full_result_with_multiple_result_keys(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {
"Users": ["User1"],
"Groups": ["Group1"],
"Marker": "m1"
}),
(None, {
"Users": ["User2"],
"Groups": ["Group2"],
"Marker": "m2"
}),
(None, {
"Users": ["User3"],
"Groups": ["Group3"],
}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(
complete, {
"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']
})
def test_next_token_with_or_expression(self):
self.operation.pagination = {
'output_token': 'NextToken or NextToken2',
'py_input_token': 'NextToken',
}
self.paginator = Paginator(self.operation)
# Verify that despite varying between NextToken and NextToken2
# we still can extract the right next tokens.
responses = [
(None, {
'NextToken': 'token1'
}),
(None, {
'NextToken2': 'token2'
}),
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
(None, {
'NextToken': 'token3',
'NextToken2': 'token4'
}),
(None, {
'not_next_token': 'foo'
}),
]
self.operation.call.side_effect = responses
actual = list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, NextToken='token1'),
mock.call(None, NextToken='token2'),
mock.call(None, NextToken='token3'),
])
def test_max_items_can_be_specified_truncates_response(self):
# We're saying we only want 4 items, but notice that the second
# page of results returns users 4-6 so we have to truncated
# part of that second page.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1", "User2", "User3"],
"Marker": "m1"
},
{
"Users": ["User4", "User5", "User6"],
"Marker": "m2"
},
{
"Users": ["User7"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 4
}).build_full_result(), {
'Users': ['User1', 'User2', 'User3', 'User4'],
'NextToken': 'm1___1'
})
def test_include_with_multiple_result_keys(self):
self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2']
self.paginator = Paginator(self.method, self.paginate_config)
self.method.side_effect = [{
'ResultKey1': ['a', 'b'],
'ResultKey2': ['u', 'v'],
'NotResultKey': 'a',
'NextToken': 'token1'
}, {
'ResultKey1': ['c', 'd'],
'ResultKey2': ['w', 'x'],
'NotResultKey': 'a',
'NextToken': 'token2'
}, {
'ResultKey1': ['e', 'f'],
'ResultKey2': ['y', 'z'],
'NotResultKey': 'a'
}]
pages = self.paginator.paginate()
actual = pages.build_full_result()
expected = {
'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'],
'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'],
'NotResultKey': 'a',
}
self.assertEqual(actual, expected)
def test_more_tokens_is_path_expression(self):
self.paginate_config = {
'more_results': 'Foo.IsTruncated',
'output_token': 'NextToken',
'py_input_token': 'NextToken',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {
'Foo': {
'IsTruncated': True
},
'NextToken': 'token1'
}),
(None, {
'Foo': {
'IsTruncated': False
},
'NextToken': 'token2'
}),
]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, NextToken='token1'),
])
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'Marker',
'input_token': 'Marker',
'result_key': ['ResultKey', 'Count', 'Log'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def get_logging_config_paginator(client):
return Paginator(
client.list_query_logging_configs, {
'input_token': 'NextToken',
'output_token': 'NextToken',
'result_key': 'QueryLoggingConfigs'
},
client.meta.service_model.operation_model('ListQueryLoggingConfigs'))
def setUp(self):
self.operation = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'py_input_token': 'NextToken',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def ls(args):
list_secrets_paginator = Paginator(method=clients.secretsmanager.list_secrets,
pagination_config=dict(result_key="SecretList",
input_token="NextToken",
output_token="NextToken",
limit_key="MaxResults"),
model=None)
page_output(tabulate(paginate(list_secrets_paginator), args))
def get_layer_version_paginator(client):
pager = Paginator(
client.list_layer_versions, {
'input_token': 'NextToken',
'output_token': 'NextToken',
'result_key': 'LayerVersions'
}, client.meta.service_model.operation_model('ListLayerVersions'))
pager.PAGE_ITERATOR_CLS = query.RetryPageIterator
return pager
def setUp(self):
self.operation = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'ResultKey',
'non_aggregate_keys': ['NotResultKey'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
'output_token':
['NextMarker or ListBucketResult.Contents[-1].Key'],
'py_input_token': 'next_marker',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": "Users"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def setUp(self):
self.method = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": ["Users", "Groups"],
'limit_key': 'MaxKeys',
}
self.paginator = Paginator(self.method, self.paginate_config)
self.endpoint = mock.Mock()
def setUp(self):
self.method = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
'output_token':
['NextMarker || ListBucketResult.Contents[-1].Key'],
'input_token': 'next_marker',
'result_key': 'Contents',
}
self.paginator = Paginator(self.method, self.paginate_config)
