class TestSearchOverResults(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'more_results': 'IsTruncated',
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [
{'Foo': [{'a': 1}, {'b': 2}],
'IsTruncated': True, 'NextToken': '1'},
{'Foo': [{'a': 3}, {'b': 4}],
'IsTruncated': True, 'NextToken': '2'},
{'Foo': [{'a': 5}], 'IsTruncated': False, 'NextToken': '3'}
]
self.method.side_effect = responses
def test_yields_non_list_values(self):
result = list(self.paginator.paginate().search('Foo[0].a'))
self.assertEqual([1, 3, 5], result)
def test_yields_individual_list_values(self):
result = list(self.paginator.paginate().search('Foo[].*[]'))
self.assertEqual([1, 2, 3, 4, 5], result)
def test_empty_when_no_match(self):
result = list(self.paginator.paginate().search('Foo[].qux'))
self.assertEqual([], result)
def test_no_yield_when_no_match_on_page(self):
result = list(self.paginator.paginate().search('Foo[].b'))
self.assertEqual([2, 4], result)
class TestIncludeResultKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'Marker',
'input_token': 'Marker',
'result_key': ['ResultKey', 'Count', 'Log'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_different_kinds_of_result_key(self):
self.method.side_effect = [
{'ResultKey': ['a'], 'Count': 1, 'Log': 'x', 'Marker': 'a'},
{'not_a_result_key': 'this page will be ignored', 'Marker': '_'},
{'ResultKey': ['b', 'c'], 'Count': 2, 'Log': 'y', 'Marker': 'b'},
{'ResultKey': ['d', 'e', 'f'], 'Count': 3, 'Log': 'z'},
]
pages = self.paginator.paginate()
expected = {
'ResultKey': ['a', 'b', 'c', 'd', 'e', 'f'],
'Count': 6,
'Log': 'xyz',
}
self.assertEqual(pages.build_full_result(), expected)
def test_result_key_is_missing(self):
self.method.side_effect = [
{'not_a_result_key': 'this page will be ignored', 'Marker': '_'},
{'neither_this_one': 'this page will be ignored, too'},
]
pages = self.paginator.paginate()
expected = {}
self.assertEqual(pages.build_full_result(), expected)
class TestMultipleInputKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_build_full_result_with_multiple_input_keys(self):
responses = [
{"Users": ["User1", "User2"], "Groups": ["Group1"],
"Marker1": "m1", "Marker2": "m2"},
{"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"},
{"Users": ["User5"], "Groups": ["Group3"]}
]
self.method.side_effect = responses
pages = self.paginator.paginate(
PaginationConfig={'MaxItems': 3})
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2'],
"NextToken": "m1___m2___1"})
def test_resume_with_multiple_input_keys(self):
responses = [
{"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"},
{"Users": ["User5"], "Groups": ["Group3"]},
]
self.method.side_effect = responses
pages = self.paginator.paginate(
PaginationConfig={'MaxItems': 1,
'StartingToken': 'm1___m2___1'})
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User4'],
"Groups": [],
"NextToken": "m3___m4"})
self.assertEqual(
self.method.call_args_list,
[mock.call(InMarker1='m1', InMarker2='m2')])
def test_result_key_exposed_on_paginator(self):
self.assertEqual(
[rk.expression for rk in self.paginator.result_keys],
['Users', 'Groups']
)
def test_result_key_exposed_on_page_iterator(self):
pages = self.paginator.paginate(MaxItems=3)
self.assertEqual(
[rk.expression for rk in pages.result_keys],
['Users', 'Groups']
)
class TestPaginatorPageSize(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": ["Users", "Groups"],
'limit_key': 'MaxKeys',
}
self.paginator = Paginator(self.method, self.paginate_config)
self.endpoint = mock.Mock()
def test_no_page_size(self):
kwargs = {'arg1': 'foo', 'arg2': 'bar'}
ref_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
pages = self.paginator.paginate(**kwargs)
pages._inject_starting_params(kwargs)
self.assertEqual(kwargs, ref_kwargs)
def test_page_size(self):
kwargs = {'arg1': 'foo', 'arg2': 'bar',
'PaginationConfig': {'PageSize': 5}}
extracted_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
# Note that ``MaxKeys`` in ``setUp()`` is the parameter used for
# the page size for pagination.
ref_kwargs = {'arg1': 'foo', 'arg2': 'bar', 'MaxKeys': 5}
pages = self.paginator.paginate(**kwargs)
pages._inject_starting_params(extracted_kwargs)
self.assertEqual(extracted_kwargs, ref_kwargs)
class TestKeyIterators(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": "Users"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_result_key_iters(self):
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
iterators = pages.result_key_iters()
self.assertEqual(len(iterators), 1)
self.assertEqual(list(iterators[0]),
["User1", "User2", "User3"])
self.assertEqual(len(pages.http_responses), 3)
def test_build_full_result_with_single_key(self):
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete, ['User1', 'User2', 'User3'])
def test_build_full_result_with_multiple_result_keys(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": ["User3"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']})
def get_layer_version_paginator(client):
pager = Paginator(
client.list_layer_versions,
{'input_token': 'NextToken',
'output_token': 'NextToken',
'result_key': 'LayerVersions'},
client.meta.service_model.operation_model('ListLayerVersions'))
pager.PAGE_ITERATOR_CLS = query.RetryPageIterator
return pager
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'ResultKey',
'non_aggregate_keys': ['NotResultKey'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
class TestFuturePaginator(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = FuturePaginator(self.method, self.paginate_config)
def test_with_page_size(self):
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(page_size=1):
users += page['Users']
self.assertEqual(
self.method.call_args_list,
[mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)]
)
def test_with_empty_markers(self):
responses = [
{"Users": ["User1"], "Marker": ""},
{"Users": ["User1"], "Marker": ""},
{"Users": ["User1"], "Marker": ""}
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate():
users += page['Users']
# We want to stop paginating if the next token is empty.
self.assertEqual(
self.method.call_args_list,
[mock.call()]
)
self.assertEqual(users, ['User1'])
def test_build_full_result_with_single_key(self):
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]}
]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
'output_token':
['NextMarker or ListBucketResult.Contents[-1].Key'],
'py_input_token': 'next_marker',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def setUp(self):
self.method = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
'output_token':
['NextMarker || ListBucketResult.Contents[-1].Key'],
'input_token': 'next_marker',
'result_key': 'Contents',
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": "Users"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def setUp(self):
self.operation = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'py_input_token': 'NextToken',
'result_key': 'ResultKey',
'non_aggregate_keys': ['NotResultKey'],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": ["Users", "Groups"],
'limit_key': 'MaxKeys',
}
self.paginator = Paginator(self.method, self.paginate_config)
self.endpoint = mock.Mock()
class TestMultipleResultKeys(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_build_full_result_with_multiple_result_keys(self):
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": ["User3"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']})
def test_build_full_result_with_different_length_result_keys(self):
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
# Then we stop getting "Users" output, but we get more "Groups"
(None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": [], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1'],
"Groups": ['Group1', 'Group2', 'Group3']})
def test_build_full_result_with_zero_length_result_key(self):
responses = [
# In this case the 'Users' key is always empty but we should
# have a 'Users' key in the output, it should just have an
# empty list for a value.
(None, {"Users": [], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": [], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": [],
"Groups": ['Group1', 'Group2', 'Group3']})
class TestIncludeResultKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'Marker',
'input_token': 'Marker',
'result_key': ['ResultKey', 'Count', 'Log'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_different_kinds_of_result_key(self):
self.method.side_effect = [
{
'ResultKey': ['a'],
'Count': 1,
'Log': 'x',
'Marker': 'a'
},
{
'not_a_result_key': 'this page will be ignored',
'Marker': '_'
},
{
'ResultKey': ['b', 'c'],
'Count': 2,
'Log': 'y',
'Marker': 'b'
},
{
'ResultKey': ['d', 'e', 'f'],
'Count': 3,
'Log': 'z'
},
]
pages = self.paginator.paginate()
expected = {
'ResultKey': ['a', 'b', 'c', 'd', 'e', 'f'],
'Count': 6,
'Log': 'xyz',
}
self.assertEqual(pages.build_full_result(), expected)
def test_result_key_is_missing(self):
self.method.side_effect = [
{
'not_a_result_key': 'this page will be ignored',
'Marker': '_'
},
{
'neither_this_one': 'this page will be ignored, too'
},
]
pages = self.paginator.paginate()
expected = {}
self.assertEqual(pages.build_full_result(), expected)
def test_max_items_can_be_specified(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(max_items=1).build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(max_items=2).build_full_result(),
{'Users': ['User1', 'User2'], 'NextToken': 'm2'})
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": ["ListBucketResults.NextKeyMarker",
"ListBucketResults.NextUploadIdMarker"],
"py_input_token": ["key_marker", "upload_id_marker"],
"result_key": 'Foo',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_max_items_can_be_specified(self):
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=1).build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=2).build_full_result(),
{'Users': ['User1', 'User2'], 'NextToken': 'm2'})
def test_include_with_nested_result_keys(self):
self.paginate_config['result_key'] = 'Result.Key'
self.paginate_config['non_aggregate_keys'] = [
'Outer',
'Result.Inner',
]
self.paginator = Paginator(self.method, self.paginate_config)
self.method.side_effect = [
# The non result keys shows hypothetical
# example. This doesn't actually happen,
# but in the case where the non result keys
# are different across pages, we use the values
# from the first page.
{
'Result': {
'Key': ['foo'],
'Inner': 'v1'
},
'Outer': 'v2',
'NextToken': 't1'
},
{
'Result': {
'Key': ['bar', 'baz'],
'Inner': 'v3'
},
'Outer': 'v4',
'NextToken': 't2'
},
{
'Result': {
'Key': ['qux'],
'Inner': 'v5'
},
'Outer': 'v6',
'NextToken': 't3'
},
]
pages = self.paginator.paginate()
actual = pages.build_full_result()
self.assertEqual(pages.non_aggregate_part, {
'Outer': 'v2',
'Result': {
'Inner': 'v1'
}
})
expected = {
'Result': {
'Key': ['foo', 'bar', 'baz', 'qux'],
'Inner': 'v1'
},
'Outer': 'v2',
}
self.assertEqual(actual, expected)
def _get_resource_paginator(self, client):
p = Paginator(
client.list_resources,
{
"input_token": "NextToken",
"output_token": "NextToken",
"result_key": "ResourceDescriptions",
},
client.meta.service_model.operation_model("ListResources"),
)
p.PAGE_ITERATOR_CLS = RetryPageIterator
return p
def setUp(self):
self.method = mock.Mock()
# This is based on Route53 pagination.
self.paginate_config = {
"output_token":
["NextRecordName", "NextRecordType", "NextRecordIdentifier"],
"input_token":
["StartRecordName", "StartRecordType", "StartRecordIdentifier"],
"result_key":
'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
class TestMultipleInputKeys(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"py_input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_build_full_result_with_multiple_input_keys(self):
responses = [
(None, {"Users": ["User1", "User2"], "Groups": ["Group1"],
"Marker1": "m1", "Marker2": "m2"}),
(None, {"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"}),
(None, {"Users": ["User5"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None, max_items=3)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2'],
"NextToken": "m1___m2___1"})
def test_resume_with_multiple_input_keys(self):
responses = [
(None, {"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"}),
(None, {"Users": ["User5"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None, max_items=1,
starting_token='m1___m2___1')
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User4'],
"Groups": [],
"NextToken": "m3___m4"})
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None, InMarker1='m1', InMarker2='m2'),])
def test_result_key_exposed_on_paginator(self):
self.assertEqual(self.paginator.result_keys, ['Users', 'Groups'])
def test_result_key_exposed_on_page_iterator(self):
pages = self.paginator.paginate(None, max_items=3)
self.assertEqual(pages.result_keys, ['Users', 'Groups'])
def test_max_items_exceeds_actual_amount(self):
# Because MaxItems=10 > number of users (3), we should just return
# all of the users.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(max_items=10).build_full_result(),
{'Users': ['User1', 'User2', 'User3']})
def test_max_items_can_be_specified(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
expected_token = encode_token({"Marker": "m1"})
self.assertEqual(
paginator.paginate(
PaginationConfig={'MaxItems': 1}).build_full_result(),
{'Users': ['User1'], 'NextToken': expected_token})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
expected_token = encode_token({"Marker": "m2"})
self.assertEqual(
paginator.paginate(
PaginationConfig={'MaxItems': 2}).build_full_result(),
{'Users': ['User1', 'User2'], 'NextToken': expected_token})
def test_max_items_exceeds_actual_amount(self):
# Because MaxItems=10 > number of users (3), we should just return
# all of the users.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=10).build_full_result(),
{'Users': ['User1', 'User2', 'User3']})
class TestFuturePaginator(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = FuturePaginator(self.method, self.paginate_config)
def test_with_page_size(self):
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(page_size=1):
users += page['Users']
self.assertEqual(self.method.call_args_list, [
mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)
])
def test_build_full_result_with_single_key(self):
responses = [{
"Users": ["User1"],
"Marker": "m1"
}, {
"Users": ["User2"],
"Marker": "m2"
}, {
"Users": ["User3"]
}]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
def test_max_items_as_strings(self):
# Some services (route53) model MaxItems as a string type.
# We need to be able to handle this case.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
# Note max_items is a string here.
paginator.paginate(None, max_items='1').build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
class TestSearchOverResults(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'more_results': 'IsTruncated',
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [{
'Foo': [{
'a': 1
}, {
'b': 2
}],
'IsTruncated': True,
'NextToken': '1'
}, {
'Foo': [{
'a': 3
}, {
'b': 4
}],
'IsTruncated': True,
'NextToken': '2'
}, {
'Foo': [{
'a': 5
}],
'IsTruncated': False,
'NextToken': '3'
}]
self.method.side_effect = responses
def test_yields_non_list_values(self):
result = list(self.paginator.paginate().search('Foo[0].a'))
self.assertEqual([1, 3, 5], result)
def test_yields_individual_list_values(self):
result = list(self.paginator.paginate().search('Foo[].*[]'))
self.assertEqual([1, 2, 3, 4, 5], result)
def test_empty_when_no_match(self):
result = list(self.paginator.paginate().search('Foo[].qux'))
self.assertEqual([], result)
def test_no_yield_when_no_match_on_page(self):
result = list(self.paginator.paginate().search('Foo[].b'))
self.assertEqual([2, 4], result)
class TestPaginatorWithPathExpressions(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
'output_token':
['NextMarker or ListBucketResult.Contents[-1].Key'],
'py_input_token': 'next_marker',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_s3_list_objects(self):
responses = [(None, {
'NextMarker': 'token1'
}), (None, {
'NextMarker': 'token2'
}), (None, {
'not_next_token': 'foo'
})]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, next_marker='token1'),
mock.call(None, next_marker='token2'),
])
def test_s3_list_object_complex(self):
responses = [(None, {
'NextMarker': 'token1'
}),
(None, {
'ListBucketResult': {
'Contents': [{
"Key": "first"
}, {
"Key": "Last"
}]
}
}), (None, {
'not_next_token': 'foo'
})]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, next_marker='token1'),
mock.call(None, next_marker='Last'),
])
class TestPaginatorWithPathExpressions(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
'output_token':
['NextMarker || ListBucketResult.Contents[-1].Key'],
'input_token': 'next_marker',
'result_key': 'Contents',
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_s3_list_objects(self):
responses = [{
'NextMarker': 'token1'
}, {
'NextMarker': 'token2'
}, {
'not_next_token': 'foo'
}]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(self.method.call_args_list, [
mock.call(),
mock.call(next_marker='token1'),
mock.call(next_marker='token2')
])
def test_s3_list_object_complex(self):
responses = [{
'NextMarker': 'token1'
}, {
'ListBucketResult': {
'Contents': [{
"Key": "first"
}, {
"Key": "Last"
}]
}
}, {
'not_next_token': 'foo'
}]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(self.method.call_args_list, [
mock.call(),
mock.call(next_marker='token1'),
mock.call(next_marker='Last')
])
def test_max_items_as_strings(self):
# Some services (route53) model MaxItems as a string type.
# We need to be able to handle this case.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
# Note MaxItems is a string here.
paginator.paginate(
PaginationConfig={'MaxItems': '1'}).build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_max_items_can_be_specified_truncates_response(self):
# We're saying we only want 4 items, but notice that the second
# page of results returns users 4-6 so we have to truncated
# part of that second page.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1", "User2", "User3"], "Marker": "m1"}),
(None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
(None, {"Users": ["User7"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=4).build_full_result(),
{'Users': ['User1', 'User2', 'User3', 'User4'],
'NextToken': 'm1___1'})
def resources(self, query=None):
client = local_session(self.manager.session_factory).client('config')
query = self.get_query_params(query)
pager = Paginator(
client.select_resource_config,
{'input_token': 'NextToken', 'output_token': 'NextToken',
'result_key': 'Results'},
client.meta.service_model.operation_model('SelectResourceConfig'))
pager.PAGE_ITERATOR_CLS = RetryPageIterator
results = []
for page in pager.paginate(Expression=query['expr']):
results.extend([
self.load_resource(json.loads(r)) for r in page['Results']])
return results
def get_paginator(self, operation_name):
"""Create a paginator for an operation.
:type operation_name: string
:param operation_name: The operation name. This is the same name
as the method name on the client. For example, if the
method name is ``create_foo``, and you'd normally invoke the
operation as ``client.create_foo(**kwargs)``, if the
``create_foo`` operation can be paginated, you can use the
call ``client.get_paginator("create_foo")``.
:raise OperationNotPageableError: Raised if the operation is not
pageable. You can use the ``client.can_paginate`` method to
check if an operation is pageable.
:rtype: L{botocore.paginate.Paginator}
:return: A paginator object.
"""
if not self.can_paginate(operation_name):
raise OperationNotPageableError(operation_name=operation_name)
else:
actual_operation_name = self._PY_TO_OP_NAME[operation_name]
paginator = Paginator(
getattr(self, operation_name),
self._cache['page_config'][actual_operation_name])
return paginator
def test_next_token_with_or_expression(self):
self.operation.pagination = {
'output_token': 'NextToken || NextToken2',
'py_input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.operation)
# Verify that despite varying between NextToken and NextToken2
# we still can extract the right next tokens.
responses = [
(None, {'NextToken': 'token1'}),
(None, {'NextToken2': 'token2'}),
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
(None, {'NextToken': 'token3', 'NextToken2': 'token4'}),
(None, {'not_next_token': 'foo'}),
]
self.operation.call.side_effect = responses
actual = list(self.paginator.paginate(None))
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None),
mock.call(None, NextToken='token1'),
mock.call(None, NextToken='token2'),
mock.call(None, NextToken='token3'),])
def test_include_with_nested_result_keys(self):
self.paginate_config['result_key'] = 'Result.Key'
self.paginate_config['non_aggregate_keys'] = [
'Outer', 'Result.Inner',
]
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
self.set_responses([
# The non result keys shows hypothetical
# example. This doesn't actually happen,
# but in the case where the non result keys
# are different across pages, we use the values
# from the first page.
{'Result': {'Key': ['foo'], 'Inner': 'v1'},
'Outer': 'v2', 'NextToken': 't1'},
{'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'},
'Outer': 'v4', 'NextToken': 't2'},
{'Result': {'Key': ['qux'], 'Inner': 'v5'},
'Outer': 'v6', 'NextToken': 't3'},
])
pages = self.paginator.paginate(None)
actual = pages.build_full_result()
self.assertEqual(pages.non_aggregate_part,
{'Outer': 'v2', 'Result': {'Inner': 'v1'}})
expected = {
'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'},
'Outer': 'v2',
}
self.assertEqual(actual, expected)
class TestMultipleTokens(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": ["ListBucketResults.NextKeyMarker",
"ListBucketResults.NextUploadIdMarker"],
"py_input_token": ["key_marker", "upload_id_marker"],
"result_key": 'Foo',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_s3_list_multipart_uploads(self):
responses = [
(None, {"Foo": [1], "ListBucketResults": {"NextKeyMarker": "key1",
"NextUploadIdMarker": "up1"}}),
(None, {"Foo": [2], "ListBucketResults": {"NextKeyMarker": "key2",
"NextUploadIdMarker": "up2"}}),
(None, {"Foo": [3], "ListBucketResults": {"NextKeyMarker": "key3",
"NextUploadIdMarker": "up3"}}),
(None, {}),
]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None),
mock.call(None, key_marker='key1', upload_id_marker='up1'),
mock.call(None, key_marker='key2', upload_id_marker='up2'),
mock.call(None, key_marker='key3', upload_id_marker='up3'),
])
def get_protections_paginator(client):
return Paginator(
client.list_protections, {
'input_token': 'NextToken',
'output_token': 'NextToken',
'result_key': 'Protections'
}, client.meta.service_model.operation_model('ListProtections'))
def setUp(self):
self.operation = mock.Mock()
# This is something like what we'd see in RDS.
self.paginate_config = {
"input_token": "Marker",
"output_token": "Marker",
"limit_key": "MaxRecords",
"result_key": "EngineDefaults.Parameters"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation, self.paginate_config)
self.responses = [
(None, {
"EngineDefaults": {
"Parameters": ["One", "Two"]
},
"Marker": "m1"
}),
(None, {
"EngineDefaults": {
"Parameters": ["Three", "Four"]
},
"Marker": "m2"
}),
(None, {
"EngineDefaults": {
"Parameters": ["Five"]
}
}),
]
def test_next_token_with_or_expression(self):
self.operation.pagination = {
'output_token': 'NextToken || NextToken2',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.operation, self.operation.pagination)
# Verify that despite varying between NextToken and NextToken2
# we still can extract the right next tokens.
responses = [
(None, {
'NextToken': 'token1'
}),
(None, {
'NextToken2': 'token2'
}),
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
(None, {
'NextToken': 'token3',
'NextToken2': 'token4'
}),
(None, {
'not_next_token': 'foo'
}),
]
self.operation.call.side_effect = responses
actual = list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, NextToken='token1'),
mock.call(None, NextToken='token2'),
mock.call(None, NextToken='token3'),
])
def test_include_with_multiple_result_keys(self):
self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2']
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation, self.paginate_config)
self.set_responses([{
'ResultKey1': ['a', 'b'],
'ResultKey2': ['u', 'v'],
'NotResultKey': 'a',
'NextToken': 'token1'
}, {
'ResultKey1': ['c', 'd'],
'ResultKey2': ['w', 'x'],
'NotResultKey': 'a',
'NextToken': 'token2'
}, {
'ResultKey1': ['e', 'f'],
'ResultKey2': ['y', 'z'],
'NotResultKey': 'a',
}])
pages = self.paginator.paginate(None)
actual = pages.build_full_result()
expected = {
'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'],
'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'],
'NotResultKey': 'a',
}
self.assertEqual(actual, expected)
class TestFuturePaginator(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = FuturePaginator(self.method, self.paginate_config)
def test_with_page_size(self):
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(page_size=1):
users += page['Users']
self.assertEqual(
self.method.call_args_list,
[mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)]
)
def test_more_tokens(self):
# Some pagination configs have a 'more_token' key that
# indicate whether or not the results are being paginated.
self.paginate_config = {
'more_results': 'IsTruncated',
'output_token': 'NextToken',
'py_input_token': 'NextToken',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {'IsTruncated': True, 'NextToken': 'token1'}),
(None, {'IsTruncated': True, 'NextToken': 'token2'}),
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
(None, {'IsTruncated': False, 'NextToken': 'token3'}),
(None, {'not_next_token': 'foo'}),
]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None),
mock.call(None, NextToken='token1'),
mock.call(None, NextToken='token2'),])
def setUp(self):
self.operation = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'py_input_token': 'NextToken',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users"
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'Marker',
'input_token': 'Marker',
'result_key': ['ResultKey', 'Count', 'Log'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_resume_next_marker_mid_page(self):
# This is a simulation of picking up from the response
# from test_max_items_can_be_specified_truncates_response
# We got the first 4 users, when we pick up we should get
# User5 - User7.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
(None, {"Users": ["User7"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, starting_token='m1___1').build_full_result(),
{'Users': ['User5', 'User6', 'User7']})
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None, Marker='m1'),
mock.call(None, Marker='m2'),])