class TestIncludeResultKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'Marker',
'input_token': 'Marker',
'result_key': ['ResultKey', 'Count', 'Log'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_different_kinds_of_result_key(self):
self.method.side_effect = [
{'ResultKey': ['a'], 'Count': 1, 'Log': 'x', 'Marker': 'a'},
{'not_a_result_key': 'this page will be ignored', 'Marker': '_'},
{'ResultKey': ['b', 'c'], 'Count': 2, 'Log': 'y', 'Marker': 'b'},
{'ResultKey': ['d', 'e', 'f'], 'Count': 3, 'Log': 'z'},
]
pages = self.paginator.paginate()
expected = {
'ResultKey': ['a', 'b', 'c', 'd', 'e', 'f'],
'Count': 6,
'Log': 'xyz',
}
self.assertEqual(pages.build_full_result(), expected)
def test_result_key_is_missing(self):
self.method.side_effect = [
{'not_a_result_key': 'this page will be ignored', 'Marker': '_'},
{'neither_this_one': 'this page will be ignored, too'},
]
pages = self.paginator.paginate()
expected = {}
self.assertEqual(pages.build_full_result(), expected)
class TestPaginatorPageSize(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": ["Users", "Groups"],
'limit_key': 'MaxKeys',
}
self.paginator = Paginator(self.method, self.paginate_config)
self.endpoint = mock.Mock()
def test_no_page_size(self):
kwargs = {'arg1': 'foo', 'arg2': 'bar'}
ref_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
pages = self.paginator.paginate(**kwargs)
pages._inject_starting_params(kwargs)
self.assertEqual(kwargs, ref_kwargs)
def test_page_size(self):
kwargs = {'arg1': 'foo', 'arg2': 'bar',
'PaginationConfig': {'PageSize': 5}}
extracted_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
# Note that ``MaxKeys`` in ``setUp()`` is the parameter used for
# the page size for pagination.
ref_kwargs = {'arg1': 'foo', 'arg2': 'bar', 'MaxKeys': 5}
pages = self.paginator.paginate(**kwargs)
pages._inject_starting_params(extracted_kwargs)
self.assertEqual(extracted_kwargs, ref_kwargs)
class TestMultipleInputKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_build_full_result_with_multiple_input_keys(self):
responses = [
{"Users": ["User1", "User2"], "Groups": ["Group1"],
"Marker1": "m1", "Marker2": "m2"},
{"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"},
{"Users": ["User5"], "Groups": ["Group3"]}
]
self.method.side_effect = responses
pages = self.paginator.paginate(
PaginationConfig={'MaxItems': 3})
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2'],
"NextToken": "m1___m2___1"})
def test_resume_with_multiple_input_keys(self):
responses = [
{"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"},
{"Users": ["User5"], "Groups": ["Group3"]},
]
self.method.side_effect = responses
pages = self.paginator.paginate(
PaginationConfig={'MaxItems': 1,
'StartingToken': 'm1___m2___1'})
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User4'],
"Groups": [],
"NextToken": "m3___m4"})
self.assertEqual(
self.method.call_args_list,
[mock.call(InMarker1='m1', InMarker2='m2')])
def test_result_key_exposed_on_paginator(self):
self.assertEqual(
[rk.expression for rk in self.paginator.result_keys],
['Users', 'Groups']
)
def test_result_key_exposed_on_page_iterator(self):
pages = self.paginator.paginate(MaxItems=3)
self.assertEqual(
[rk.expression for rk in pages.result_keys],
['Users', 'Groups']
)
class TestSearchOverResults(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'more_results': 'IsTruncated',
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [
{'Foo': [{'a': 1}, {'b': 2}],
'IsTruncated': True, 'NextToken': '1'},
{'Foo': [{'a': 3}, {'b': 4}],
'IsTruncated': True, 'NextToken': '2'},
{'Foo': [{'a': 5}], 'IsTruncated': False, 'NextToken': '3'}
]
self.method.side_effect = responses
def test_yields_non_list_values(self):
result = list(self.paginator.paginate().search('Foo[0].a'))
self.assertEqual([1, 3, 5], result)
def test_yields_individual_list_values(self):
result = list(self.paginator.paginate().search('Foo[].*[]'))
self.assertEqual([1, 2, 3, 4, 5], result)
def test_empty_when_no_match(self):
result = list(self.paginator.paginate().search('Foo[].qux'))
self.assertEqual([], result)
def test_no_yield_when_no_match_on_page(self):
result = list(self.paginator.paginate().search('Foo[].b'))
self.assertEqual([2, 4], result)
class TestKeyIterators(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": "Users"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_result_key_iters(self):
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
iterators = pages.result_key_iters()
self.assertEqual(len(iterators), 1)
self.assertEqual(list(iterators[0]),
["User1", "User2", "User3"])
self.assertEqual(len(pages.http_responses), 3)
def test_build_full_result_with_single_key(self):
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete, ['User1', 'User2', 'User3'])
def test_build_full_result_with_multiple_result_keys(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": ["User3"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']})
class TestFuturePaginator(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = FuturePaginator(self.method, self.paginate_config)
def test_with_page_size(self):
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(page_size=1):
users += page['Users']
self.assertEqual(
self.method.call_args_list,
[mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)]
)
def test_with_empty_markers(self):
responses = [
{"Users": ["User1"], "Marker": ""},
{"Users": ["User1"], "Marker": ""},
{"Users": ["User1"], "Marker": ""}
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate():
users += page['Users']
# We want to stop paginating if the next token is empty.
self.assertEqual(
self.method.call_args_list,
[mock.call()]
)
self.assertEqual(users, ['User1'])
def test_build_full_result_with_single_key(self):
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]}
]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
class TestMultipleResultKeys(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_build_full_result_with_multiple_result_keys(self):
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": ["User3"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']})
def test_build_full_result_with_different_length_result_keys(self):
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
# Then we stop getting "Users" output, but we get more "Groups"
(None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": [], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1'],
"Groups": ['Group1', 'Group2', 'Group3']})
def test_build_full_result_with_zero_length_result_key(self):
responses = [
# In this case the 'Users' key is always empty but we should
# have a 'Users' key in the output, it should just have an
# empty list for a value.
(None, {"Users": [], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": [], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": [],
"Groups": ['Group1', 'Group2', 'Group3']})
class TestMultipleInputKeys(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"py_input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_build_full_result_with_multiple_input_keys(self):
responses = [
(None, {"Users": ["User1", "User2"], "Groups": ["Group1"],
"Marker1": "m1", "Marker2": "m2"}),
(None, {"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"}),
(None, {"Users": ["User5"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None, max_items=3)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2'],
"NextToken": "m1___m2___1"})
def test_resume_with_multiple_input_keys(self):
responses = [
(None, {"Users": ["User3", "User4"], "Groups": ["Group2"],
"Marker1": "m3", "Marker2": "m4"}),
(None, {"Users": ["User5"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None, max_items=1,
starting_token='m1___m2___1')
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User4'],
"Groups": [],
"NextToken": "m3___m4"})
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None, InMarker1='m1', InMarker2='m2'),])
def test_result_key_exposed_on_paginator(self):
self.assertEqual(self.paginator.result_keys, ['Users', 'Groups'])
def test_result_key_exposed_on_page_iterator(self):
pages = self.paginator.paginate(None, max_items=3)
self.assertEqual(pages.result_keys, ['Users', 'Groups'])
class TestFuturePaginator(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = FuturePaginator(self.method, self.paginate_config)
def test_with_page_size(self):
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(page_size=1):
users += page['Users']
self.assertEqual(
self.method.call_args_list,
[mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)]
)
class TestMultipleTokens(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": ["ListBucketResults.NextKeyMarker",
"ListBucketResults.NextUploadIdMarker"],
"py_input_token": ["key_marker", "upload_id_marker"],
"result_key": 'Foo',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_s3_list_multipart_uploads(self):
responses = [
(None, {"Foo": [1], "ListBucketResults": {"NextKeyMarker": "key1",
"NextUploadIdMarker": "up1"}}),
(None, {"Foo": [2], "ListBucketResults": {"NextKeyMarker": "key2",
"NextUploadIdMarker": "up2"}}),
(None, {"Foo": [3], "ListBucketResults": {"NextKeyMarker": "key3",
"NextUploadIdMarker": "up3"}}),
(None, {}),
]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None),
mock.call(None, key_marker='key1', upload_id_marker='up1'),
mock.call(None, key_marker='key2', upload_id_marker='up2'),
mock.call(None, key_marker='key3', upload_id_marker='up3'),
])
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=2).build_full_result(),
{'Users': ['User1', 'User2'], 'NextToken': 'm2'})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(max_items=2).build_full_result(),
{'Users': ['User1', 'User2'], 'NextToken': 'm2'})
def test_max_items_can_be_specified(self):
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=1).build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_max_items_can_be_specified(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(max_items=1).build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_max_items_can_be_specified(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
expected_token = encode_token({"Marker": "m1"})
self.assertEqual(
paginator.paginate(
PaginationConfig={'MaxItems': 1}).build_full_result(),
{'Users': ['User1'], 'NextToken': expected_token})
def test_max_items_exceeds_actual_amount(self):
# Because MaxItems=10 > number of users (3), we should just return
# all of the users.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(max_items=10).build_full_result(),
{'Users': ['User1', 'User2', 'User3']})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
expected_token = encode_token({"Marker": "m2"})
self.assertEqual(
paginator.paginate(
PaginationConfig={'MaxItems': 2}).build_full_result(),
{'Users': ['User1', 'User2'], 'NextToken': expected_token})
class TestPaginatorWithPathExpressions(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
'output_token': [
'NextMarker || ListBucketResult.Contents[-1].Key'],
'py_input_token': 'next_marker',
'result_key': 'Contents',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_s3_list_objects(self):
responses = [
(None, {'NextMarker': 'token1'}),
(None, {'NextMarker': 'token2'}),
(None, {'not_next_token': 'foo'})]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None),
mock.call(None, next_marker='token1'),
mock.call(None, next_marker='token2'),])
def test_s3_list_object_complex(self):
responses = [
(None, {'NextMarker': 'token1'}),
(None, {'ListBucketResult': {
'Contents': [{"Key": "first"}, {"Key": "Last"}]}}),
(None, {'not_next_token': 'foo'})]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None),
mock.call(None, next_marker='token1'),
mock.call(None, next_marker='Last'),])
def test_max_items_exceeds_actual_amount(self):
# Because MaxItems=10 > number of users (3), we should just return
# all of the users.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=10).build_full_result(),
{'Users': ['User1', 'User2', 'User3']})
class TestExpressionKeyIterators(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something like what we'd see in RDS.
self.paginate_config = {
"py_input_token": "Marker",
"output_token": "Marker",
"limit_key": "MaxRecords",
"result_key": "EngineDefaults.Parameters"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
self.responses = [
(None, {
"EngineDefaults": {"Parameters": ["One", "Two"]
}, "Marker": "m1"}),
(None, {
"EngineDefaults": {"Parameters": ["Three", "Four"]
}, "Marker": "m2"}),
(None, {"EngineDefaults": {"Parameters": ["Five"]}}),
]
def test_result_key_iters(self):
self.operation.call.side_effect = self.responses
pages = self.paginator.paginate(None)
iterators = pages.result_key_iters()
self.assertEqual(len(iterators), 1)
self.assertEqual(list(iterators[0]),
['One', 'Two', 'Three', 'Four', 'Five'])
def test_build_full_result_with_single_key(self):
self.operation.call.side_effect = self.responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete, {
'EngineDefaults': {
'Parameters': ['One', 'Two', 'Three', 'Four', 'Five']
},
})
def test_max_items_as_strings(self):
# Some services (route53) model MaxItems as a string type.
# We need to be able to handle this case.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
# Note max_items is a string here.
paginator.paginate(None, max_items='1').build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_max_items_as_strings(self):
# Some services (route53) model MaxItems as a string type.
# We need to be able to handle this case.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1"], "Marker": "m1"},
{"Users": ["User2"], "Marker": "m2"},
{"Users": ["User3"]},
]
self.method.side_effect = responses
self.assertEqual(
# Note MaxItems is a string here.
paginator.paginate(
PaginationConfig={'MaxItems': '1'}).build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_max_items_can_be_specified_truncates_response(self):
# We're saying we only want 4 items, but notice that the second
# page of results returns users 4-6 so we have to truncated
# part of that second page.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1", "User2", "User3"], "Marker": "m1"}),
(None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
(None, {"Users": ["User7"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=4).build_full_result(),
{'Users': ['User1', 'User2', 'User3', 'User4'],
'NextToken': 'm1___1'})
class TestOptionalTokens(unittest.TestCase):
"""
Tests a paginator with an optional output token.
The Route53 ListResourceRecordSets paginator includes three output tokens,
one of which only appears in certain records. If this gets left in the
request params from a previous page, the API will skip over a record.
"""
def setUp(self):
self.method = mock.Mock()
# This is based on Route53 pagination.
self.paginate_config = {
"output_token": ["NextRecordName",
"NextRecordType",
"NextRecordIdentifier"],
"input_token": ["StartRecordName",
"StartRecordType",
"StartRecordIdentifier"],
"result_key": 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_clean_token(self):
responses = [
{"Foo": [1],
"IsTruncated": True,
"NextRecordName": "aaa.example.com",
"NextRecordType": "A",
"NextRecordIdentifier": "id"},
{"Foo": [2],
"IsTruncated": True,
"NextRecordName": "bbb.example.com",
"NextRecordType": "A"},
{"Foo": [3],
"IsTruncated": False},
]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(
self.method.call_args_list,
[mock.call(),
mock.call(StartRecordName='aaa.example.com', StartRecordType='A',
StartRecordIdentifier='id'),
mock.call(StartRecordName='bbb.example.com', StartRecordType='A')
])
def test_resume_next_marker_mid_page(self):
# This is a simulation of picking up from the response
# from test_max_items_can_be_specified_truncates_response
# We got the first 4 users, when we pick up we should get
# User5 - User7.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
(None, {"Users": ["User7"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, starting_token='m1___1').build_full_result(),
{'Users': ['User5', 'User6', 'User7']})
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None, Marker='m1'),
mock.call(None, Marker='m2'),])
def test_max_items_can_be_specified_truncates_response(self):
# We're saying we only want 4 items, but notice that the second
# page of results returns users 4-6 so we have to truncated
# part of that second page.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User1", "User2", "User3"], "Marker": "m1"},
{"Users": ["User4", "User5", "User6"], "Marker": "m2"},
{"Users": ["User7"]},
]
self.method.side_effect = responses
expected_token = encode_token(
{"Marker": "m1", "boto_truncate_amount": 1})
self.assertEqual(
paginator.paginate(
PaginationConfig={'MaxItems': 4}).build_full_result(),
{'Users': ['User1', 'User2', 'User3', 'User4'],
'NextToken': expected_token})
def test_resume_next_marker_mid_page(self):
# This is a simulation of picking up from the response
# from test_MaxItems_can_be_specified_truncates_response
# We got the first 4 users, when we pick up we should get
# User5 - User7.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{"Users": ["User4", "User5", "User6"], "Marker": "m2"},
{"Users": ["User7"]},
]
self.method.side_effect = responses
pagination_config = {'StartingToken': 'm1___1'}
self.assertEqual(
paginator.paginate(
PaginationConfig=pagination_config).build_full_result(),
{'Users': ['User5', 'User6', 'User7']})
self.assertEqual(
self.method.call_args_list,
[mock.call(Marker='m1'),
mock.call(Marker='m2')])
def resources(self, query=None):
client = local_session(self.manager.session_factory).client('config')
query = self.get_query_params(query)
pager = Paginator(
client.select_resource_config,
{'input_token': 'NextToken', 'output_token': 'NextToken',
'result_key': 'Results'},
client.meta.service_model.operation_model('SelectResourceConfig'))
pager.PAGE_ITERATOR_CLS = RetryPageIterator
results = []
for page in pager.paginate(Expression=query['expr']):
results.extend([
self.load_resource(json.loads(r)) for r in page['Results']])
# Config arbitrarily breaks which resource types its supports for query/select
# on any given day, if we don't have a user defined query, then fallback
# to iteration mode.
if not results and query == self.get_query_params({}):
results = self.get_listed_resources(client)
return results
def test_max_items_exceeds_actual_amount(self):
# Because MaxItems=10 > number of users (3), we should just return
# all of the users.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 10
}).build_full_result(), {'Users': ['User1', 'User2', 'User3']})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 2
}).build_full_result(), {
'Users': ['User1', 'User2'],
'NextToken': 'm2'
})
def test_max_items_can_be_specified(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 1
}).build_full_result(), {
'Users': ['User1'],
'NextToken': 'm1'
})
def test_resume_next_marker_mid_page(self):
# This is a simulation of picking up from the response
# from test_MaxItems_can_be_specified_truncates_response
# We got the first 4 users, when we pick up we should get
# User5 - User7.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User4", "User5", "User6"],
"Marker": "m2"
},
{
"Users": ["User7"]
},
]
self.method.side_effect = responses
pagination_config = {'StartingToken': 'm1___1'}
self.assertEqual(
paginator.paginate(
PaginationConfig=pagination_config).build_full_result(),
{'Users': ['User5', 'User6', 'User7']})
self.assertEqual(self.method.call_args_list,
[mock.call(Marker='m1'),
mock.call(Marker='m2')])
class TestKeyIterators(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": "Users"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_result_key_iters(self):
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
iterators = pages.result_key_iters()
self.assertEqual(len(iterators), 1)
self.assertEqual(list(iterators[0]),
["User1", "User2", "User3"])
def test_build_full_result_with_single_key(self):
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
def test_max_items_can_be_specified(self):
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=1).build_full_result(),
{'Users': ['User1'], 'NextToken': 'm1'})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=2).build_full_result(),
{'Users': ['User1', 'User2'], 'NextToken': 'm2'})
def test_max_items_can_be_specified_truncates_response(self):
# We're saying we only want 4 items, but notice that the second
# page of results returns users 4-6 so we have to truncated
# part of that second page.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1", "User2", "User3"], "Marker": "m1"}),
(None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
(None, {"Users": ["User7"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=4).build_full_result(),
{'Users': ['User1', 'User2', 'User3', 'User4'],
'NextToken': 'm1___1'})
def test_resume_next_marker_mid_page(self):
# This is a simulation of picking up from the response
# from test_max_items_can_be_specified_truncates_response
# We got the first 4 users, when we pick up we should get
# User5 - User7.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}),
(None, {"Users": ["User7"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, starting_token='m1___1').build_full_result(),
{'Users': ['User5', 'User6', 'User7']})
self.assertEqual(
self.operation.call.call_args_list,
[mock.call(None, Marker='m1'),
mock.call(None, Marker='m2'),])
def test_max_items_exceeds_actual_amount(self):
# Because MaxItems=10 > number of users (3), we should just return
# all of the users.
paginator = Paginator(self.operation)
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
self.assertEqual(
paginator.paginate(None, max_items=10).build_full_result(),
{'Users': ['User1', 'User2', 'User3']})
def test_bad_input_tokens(self):
responses = [
(None, {"Users": ["User1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Marker": "m2"}),
(None, {"Users": ["User3"]}),
]
self.operation.call.side_effect = responses
with self.assertRaisesRegexp(ValueError, 'Bad starting token'):
self.paginator.paginate(
None, starting_token='bad___notanint').build_full_result()
class TestPagination(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_result_key_available(self):
self.assertEqual([rk.expression for rk in self.paginator.result_keys],
['Foo'])
def test_no_next_token(self):
response = {'not_the_next_token': 'foobar'}
self.method.return_value = response
actual = list(self.paginator.paginate())
self.assertEqual(actual, [{'not_the_next_token': 'foobar'}])
def test_next_token_in_response(self):
responses = [{
'NextToken': 'token1'
}, {
'NextToken': 'token2'
}, {
'not_next_token': 'foo'
}]
self.method.side_effect = responses
actual = list(self.paginator.paginate())
self.assertEqual(actual, responses)
# The first call has no next token, the second and third call should
# have 'token1' and 'token2' respectively.
self.assertEqual(self.method.call_args_list, [
mock.call(),
mock.call(NextToken='token1'),
mock.call(NextToken='token2')
])
def test_any_passed_in_args_are_unmodified(self):
responses = [{
'NextToken': 'token1'
}, {
'NextToken': 'token2'
}, {
'not_next_token': 'foo'
}]
self.method.side_effect = responses
actual = list(self.paginator.paginate(Foo='foo', Bar='bar'))
self.assertEqual(actual, responses)
self.assertEqual(self.method.call_args_list, [
mock.call(Foo='foo', Bar='bar'),
mock.call(Foo='foo', Bar='bar', NextToken='token1'),
mock.call(Foo='foo', Bar='bar', NextToken='token2')
])
def test_exception_raised_if_same_next_token(self):
responses = [{
'NextToken': 'token1'
}, {
'NextToken': 'token2'
}, {
'NextToken': 'token2'
}]
self.method.side_effect = responses
with self.assertRaises(PaginationError):
list(self.paginator.paginate())
def test_next_token_with_or_expression(self):
self.pagination_config = {
'output_token': 'NextToken || NextToken2',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.pagination_config)
# Verify that despite varying between NextToken and NextToken2
# we still can extract the right next tokens.
responses = [
{
'NextToken': 'token1'
},
{
'NextToken2': 'token2'
},
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
{
'NextToken': 'token3',
'NextToken2': 'token4'
},
{
'not_next_token': 'foo'
},
]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(self.method.call_args_list, [
mock.call(),
mock.call(NextToken='token1'),
mock.call(NextToken='token2'),
mock.call(NextToken='token3')
])
def test_more_tokens(self):
# Some pagination configs have a 'more_token' key that
# indicate whether or not the results are being paginated.
self.paginate_config = {
'more_results': 'IsTruncated',
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Foo',
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [
{
'Foo': [1],
'IsTruncated': True,
'NextToken': 'token1'
},
{
'Foo': [2],
'IsTruncated': True,
'NextToken': 'token2'
},
{
'Foo': [3],
'IsTruncated': False,
'NextToken': 'token3'
},
{
'Foo': [4],
'not_next_token': 'foo'
},
]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(self.method.call_args_list, [
mock.call(),
mock.call(NextToken='token1'),
mock.call(NextToken='token2')
])
def test_more_tokens_is_path_expression(self):
self.paginate_config = {
'more_results': 'Foo.IsTruncated',
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'Bar',
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [
{
'Foo': {
'IsTruncated': True
},
'NextToken': 'token1'
},
{
'Foo': {
'IsTruncated': False
},
'NextToken': 'token2'
},
]
self.method.side_effect = responses
list(self.paginator.paginate())
self.assertEqual(
self.method.call_args_list,
[mock.call(), mock.call(NextToken='token1')])
def test_page_size(self):
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate(PaginationConfig={'PageSize': 1}):
users += page['Users']
self.assertEqual(self.method.call_args_list, [
mock.call(MaxKeys=1),
mock.call(Marker='m1', MaxKeys=1),
mock.call(Marker='m2', MaxKeys=1)
])
def test_with_empty_markers(self):
responses = [{
"Users": ["User1"],
"Marker": ""
}, {
"Users": ["User1"],
"Marker": ""
}, {
"Users": ["User1"],
"Marker": ""
}]
self.method.side_effect = responses
users = []
for page in self.paginator.paginate():
users += page['Users']
# We want to stop paginating if the next token is empty.
self.assertEqual(self.method.call_args_list, [mock.call()])
self.assertEqual(users, ['User1'])
def test_build_full_result_with_single_key(self):
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users",
"limit_key": "MaxKeys",
}
self.paginator = Paginator(self.method, self.paginate_config)
responses = [{
"Users": ["User1"],
"Marker": "m1"
}, {
"Users": ["User2"],
"Marker": "m2"
}, {
"Users": ["User3"]
}]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
class TestMultipleResultKeys(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_build_full_result_with_multiple_result_keys(self):
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": ["User3"], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']})
def test_build_full_result_with_different_length_result_keys(self):
responses = [
(None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}),
# Then we stop getting "Users" output, but we get more "Groups"
(None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": [], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ['User1'],
"Groups": ['Group1', 'Group2', 'Group3']})
def test_build_full_result_with_zero_length_result_key(self):
responses = [
# In this case the 'Users' key is always empty but we should
# have a 'Users' key in the output, it should just have an
# empty list for a value.
(None, {"Users": [], "Groups": ["Group1"], "Marker": "m1"}),
(None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}),
(None, {"Users": [], "Groups": ["Group3"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": [],
"Groups": ['Group1', 'Group2', 'Group3']})
def test_build_result_with_secondary_keys(self):
responses = [
(None, {"Users": ["User1", "User2"],
"Groups": ["Group1", "Group2"],
"Marker": "m1"}),
(None, {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"}),
(None, {"Users": ["User4"], "Groups": ["Group4"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None, max_items=1)
complete = pages.build_full_result()
self.assertEqual(complete,
{"Users": ["User1"], "Groups": ["Group1", "Group2"],
"NextToken": "None___1"})
def test_resume_with_secondary_keys(self):
# This is simulating a continutation of the previous test,
# test_build_result_with_secondary_keys. We use the
# token specified in the response "None___1" to continue where we
# left off.
responses = [
(None, {"Users": ["User1", "User2"],
"Groups": ["Group1", "Group2"],
"Marker": "m1"}),
(None, {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"}),
(None, {"Users": ["User4"], "Groups": ["Group4"], }),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None, max_items=1,
starting_token="None___1")
complete = pages.build_full_result()
# Note that the secondary keys ("Groups") are all truncated because
# they were in the original (first) response.
self.assertEqual(complete,
{"Users": ["User2"], "Groups": [],
"NextToken": "m1"})
class TestKeyIterators(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": "Users"
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_result_key_iters(self):
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate()
iterators = pages.result_key_iters()
self.assertEqual(len(iterators), 1)
self.assertEqual(list(iterators[0]), ["User1", "User2", "User3"])
def test_build_full_result_with_single_key(self):
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})
def test_max_items_can_be_specified(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 1
}).build_full_result(), {
'Users': ['User1'],
'NextToken': 'm1'
})
def test_max_items_as_strings(self):
# Some services (route53) model MaxItems as a string type.
# We need to be able to handle this case.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
# Note MaxItems is a string here.
paginator.paginate(PaginationConfig={
'MaxItems': '1'
}).build_full_result(),
{
'Users': ['User1'],
'NextToken': 'm1'
})
def test_next_token_on_page_boundary(self):
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 2
}).build_full_result(), {
'Users': ['User1', 'User2'],
'NextToken': 'm2'
})
def test_max_items_can_be_specified_truncates_response(self):
# We're saying we only want 4 items, but notice that the second
# page of results returns users 4-6 so we have to truncated
# part of that second page.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1", "User2", "User3"],
"Marker": "m1"
},
{
"Users": ["User4", "User5", "User6"],
"Marker": "m2"
},
{
"Users": ["User7"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 4
}).build_full_result(), {
'Users': ['User1', 'User2', 'User3', 'User4'],
'NextToken': 'm1___1'
})
def test_resume_next_marker_mid_page(self):
# This is a simulation of picking up from the response
# from test_MaxItems_can_be_specified_truncates_response
# We got the first 4 users, when we pick up we should get
# User5 - User7.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User4", "User5", "User6"],
"Marker": "m2"
},
{
"Users": ["User7"]
},
]
self.method.side_effect = responses
pagination_config = {'StartingToken': 'm1___1'}
self.assertEqual(
paginator.paginate(
PaginationConfig=pagination_config).build_full_result(),
{'Users': ['User5', 'User6', 'User7']})
self.assertEqual(self.method.call_args_list,
[mock.call(Marker='m1'),
mock.call(Marker='m2')])
def test_max_items_exceeds_actual_amount(self):
# Because MaxItems=10 > number of users (3), we should just return
# all of the users.
paginator = Paginator(self.method, self.paginate_config)
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
self.assertEqual(
paginator.paginate(PaginationConfig={
'MaxItems': 10
}).build_full_result(), {'Users': ['User1', 'User2', 'User3']})
def test_bad_input_tokens(self):
responses = [
{
"Users": ["User1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Marker": "m2"
},
{
"Users": ["User3"]
},
]
self.method.side_effect = responses
with self.assertRaisesRegexp(ValueError, 'Bad starting token'):
pagination_config = {'StartingToken': 'bad___notanint'}
self.paginator.paginate(
PaginationConfig=pagination_config).build_full_result()
class TestIncludeNonResultKeys(unittest.TestCase):
maxDiff = None
def setUp(self):
self.method = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'input_token': 'NextToken',
'result_key': 'ResultKey',
'non_aggregate_keys': ['NotResultKey'],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_include_non_aggregate_keys(self):
self.method.side_effect = [
{
'ResultKey': ['foo'],
'NotResultKey': 'a',
'NextToken': 't1'
},
{
'ResultKey': ['bar'],
'NotResultKey': 'a',
'NextToken': 't2'
},
{
'ResultKey': ['baz'],
'NotResultKey': 'a'
},
]
pages = self.paginator.paginate()
actual = pages.build_full_result()
self.assertEqual(pages.non_aggregate_part, {'NotResultKey': 'a'})
expected = {
'ResultKey': ['foo', 'bar', 'baz'],
'NotResultKey': 'a',
}
self.assertEqual(actual, expected)
def test_include_with_multiple_result_keys(self):
self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2']
self.paginator = Paginator(self.method, self.paginate_config)
self.method.side_effect = [{
'ResultKey1': ['a', 'b'],
'ResultKey2': ['u', 'v'],
'NotResultKey': 'a',
'NextToken': 'token1'
}, {
'ResultKey1': ['c', 'd'],
'ResultKey2': ['w', 'x'],
'NotResultKey': 'a',
'NextToken': 'token2'
}, {
'ResultKey1': ['e', 'f'],
'ResultKey2': ['y', 'z'],
'NotResultKey': 'a'
}]
pages = self.paginator.paginate()
actual = pages.build_full_result()
expected = {
'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'],
'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'],
'NotResultKey': 'a',
}
self.assertEqual(actual, expected)
def test_include_with_nested_result_keys(self):
self.paginate_config['result_key'] = 'Result.Key'
self.paginate_config['non_aggregate_keys'] = [
'Outer',
'Result.Inner',
]
self.paginator = Paginator(self.method, self.paginate_config)
self.method.side_effect = [
# The non result keys shows hypothetical
# example. This doesn't actually happen,
# but in the case where the non result keys
# are different across pages, we use the values
# from the first page.
{
'Result': {
'Key': ['foo'],
'Inner': 'v1'
},
'Outer': 'v2',
'NextToken': 't1'
},
{
'Result': {
'Key': ['bar', 'baz'],
'Inner': 'v3'
},
'Outer': 'v4',
'NextToken': 't2'
},
{
'Result': {
'Key': ['qux'],
'Inner': 'v5'
},
'Outer': 'v6',
'NextToken': 't3'
},
]
pages = self.paginator.paginate()
actual = pages.build_full_result()
self.assertEqual(pages.non_aggregate_part, {
'Outer': 'v2',
'Result': {
'Inner': 'v1'
}
})
expected = {
'Result': {
'Key': ['foo', 'bar', 'baz', 'qux'],
'Inner': 'v1'
},
'Outer': 'v2',
}
self.assertEqual(actual, expected)
class TestMultipleResultKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_build_full_result_with_multiple_result_keys(self):
responses = [
{
"Users": ["User1"],
"Groups": ["Group1"],
"Marker": "m1"
},
{
"Users": ["User2"],
"Groups": ["Group2"],
"Marker": "m2"
},
{
"Users": ["User3"],
"Groups": ["Group3"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(
complete, {
"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']
})
def test_build_full_result_with_different_length_result_keys(self):
responses = [
{
"Users": ["User1"],
"Groups": ["Group1"],
"Marker": "m1"
},
# Then we stop getting "Users" output, but we get more "Groups"
{
"Users": [],
"Groups": ["Group2"],
"Marker": "m2"
},
{
"Users": [],
"Groups": ["Group3"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(complete, {
"Users": ['User1'],
"Groups": ['Group1', 'Group2', 'Group3']
})
def test_build_full_result_with_zero_length_result_key(self):
responses = [
# In this case the 'Users' key is always empty but we should
# have a 'Users' key in the output, it should just have an
# empty list for a value.
{
"Users": [],
"Groups": ["Group1"],
"Marker": "m1"
},
{
"Users": [],
"Groups": ["Group2"],
"Marker": "m2"
},
{
"Users": [],
"Groups": ["Group3"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate()
complete = pages.build_full_result()
self.assertEqual(complete, {
"Users": [],
"Groups": ['Group1', 'Group2', 'Group3']
})
def test_build_result_with_secondary_keys(self):
responses = [
{
"Users": ["User1", "User2"],
"Groups": ["Group1", "Group2"],
"Marker": "m1"
},
{
"Users": ["User3"],
"Groups": ["Group3"],
"Marker": "m2"
},
{
"Users": ["User4"],
"Groups": ["Group4"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate(PaginationConfig={'MaxItems': 1})
complete = pages.build_full_result()
self.assertEqual(
complete, {
"Users": ["User1"],
"Groups": ["Group1", "Group2"],
"NextToken": "None___1"
})
def test_resume_with_secondary_keys(self):
# This is simulating a continutation of the previous test,
# test_build_result_with_secondary_keys. We use the
# token specified in the response "None___1" to continue where we
# left off.
responses = [
{
"Users": ["User1", "User2"],
"Groups": ["Group1", "Group2"],
"Marker": "m1"
},
{
"Users": ["User3"],
"Groups": ["Group3"],
"Marker": "m2"
},
{
"Users": ["User4"],
"Groups": ["Group4"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate(PaginationConfig={
'MaxItems': 1,
'StartingToken': "None___1"
})
complete = pages.build_full_result()
# Note that the secondary keys ("Groups") are all truncated because
# they were in the original (first) response.
self.assertEqual(complete, {
"Users": ["User2"],
"Groups": [],
"NextToken": "m1"
})
class TestKeyIterators(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": "Users"
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_result_key_iters(self):
responses = [
(None, {
"Users": ["User1"],
"Marker": "m1"
}),
(None, {
"Users": ["User2"],
"Marker": "m2"
}),
(None, {
"Users": ["User3"]
}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
iterators = pages.result_key_iters()
self.assertEqual(len(iterators), 1)
self.assertEqual(list(iterators[0]), ["User1", "User2", "User3"])
self.assertEqual(len(pages.http_responses), 3)
def test_build_full_result_with_single_key(self):
responses = [
(None, {
"Users": ["User1"],
"Marker": "m1"
}),
(None, {
"Users": ["User2"],
"Marker": "m2"
}),
(None, {
"Users": ["User3"]
}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(complete, ['User1', 'User2', 'User3'])
def test_build_full_result_with_multiple_result_keys(self):
self.operation = mock.Mock()
# This is something we'd see in s3 pagination.
self.paginate_config = {
"output_token": "Marker",
"py_input_token": "Marker",
"result_key": ["Users", "Groups"],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {
"Users": ["User1"],
"Groups": ["Group1"],
"Marker": "m1"
}),
(None, {
"Users": ["User2"],
"Groups": ["Group2"],
"Marker": "m2"
}),
(None, {
"Users": ["User3"],
"Groups": ["Group3"],
}),
]
self.operation.call.side_effect = responses
pages = self.paginator.paginate(None)
complete = pages.build_full_result()
self.assertEqual(
complete, {
"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2', 'Group3']
})
class TestPagination(unittest.TestCase):
def setUp(self):
self.operation = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'py_input_token': 'NextToken',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def test_no_next_token(self):
response = {'not_the_next_token': 'foobar'}
self.operation.call.return_value = None, response
actual = list(self.paginator.paginate(None))
self.assertEqual(actual, [(None, {'not_the_next_token': 'foobar'})])
def test_next_token_in_response(self):
responses = [(None, {
'NextToken': 'token1'
}), (None, {
'NextToken': 'token2'
}), (None, {
'not_next_token': 'foo'
})]
self.operation.call.side_effect = responses
actual = list(self.paginator.paginate(None))
self.assertEqual(actual, responses)
# The first call has no next token, the second and third call should
# have 'token1' and 'token2' respectively.
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, NextToken='token1'),
mock.call(None, NextToken='token2')
])
def test_any_passed_in_args_are_unmodified(self):
responses = [(None, {
'NextToken': 'token1'
}), (None, {
'NextToken': 'token2'
}), (None, {
'not_next_token': 'foo'
})]
self.operation.call.side_effect = responses
actual = list(self.paginator.paginate(None, Foo='foo', Bar='bar'))
self.assertEqual(actual, responses)
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None, Foo='foo', Bar='bar'),
mock.call(None, Foo='foo', Bar='bar', NextToken='token1'),
mock.call(None, Foo='foo', Bar='bar', NextToken='token2')
])
def test_exception_raised_if_same_next_token(self):
responses = [(None, {
'NextToken': 'token1'
}), (None, {
'NextToken': 'token2'
}), (None, {
'NextToken': 'token2'
})]
self.operation.call.side_effect = responses
with self.assertRaises(PaginationError):
list(self.paginator.paginate(None))
def test_next_token_with_or_expression(self):
self.operation.pagination = {
'output_token': 'NextToken or NextToken2',
'py_input_token': 'NextToken',
}
self.paginator = Paginator(self.operation)
# Verify that despite varying between NextToken and NextToken2
# we still can extract the right next tokens.
responses = [
(None, {
'NextToken': 'token1'
}),
(None, {
'NextToken2': 'token2'
}),
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
(None, {
'NextToken': 'token3',
'NextToken2': 'token4'
}),
(None, {
'not_next_token': 'foo'
}),
]
self.operation.call.side_effect = responses
actual = list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, NextToken='token1'),
mock.call(None, NextToken='token2'),
mock.call(None, NextToken='token3'),
])
def test_more_tokens(self):
# Some pagination configs have a 'more_token' key that
# indicate whether or not the results are being paginated.
self.paginate_config = {
'more_results': 'IsTruncated',
'output_token': 'NextToken',
'py_input_token': 'NextToken',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {
'IsTruncated': True,
'NextToken': 'token1'
}),
(None, {
'IsTruncated': True,
'NextToken': 'token2'
}),
# The first match found wins, so because NextToken is
# listed before NextToken2 in the 'output_tokens' config,
# 'token3' is chosen over 'token4'.
(None, {
'IsTruncated': False,
'NextToken': 'token3'
}),
(None, {
'not_next_token': 'foo'
}),
]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, NextToken='token1'),
mock.call(None, NextToken='token2'),
])
def test_more_tokens_is_path_expression(self):
self.paginate_config = {
'more_results': 'Foo.IsTruncated',
'output_token': 'NextToken',
'py_input_token': 'NextToken',
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
responses = [
(None, {
'Foo': {
'IsTruncated': True
},
'NextToken': 'token1'
}),
(None, {
'Foo': {
'IsTruncated': False
},
'NextToken': 'token2'
}),
]
self.operation.call.side_effect = responses
list(self.paginator.paginate(None))
self.assertEqual(self.operation.call.call_args_list, [
mock.call(None),
mock.call(None, NextToken='token1'),
])
class TestIncludeNonResultKeys(unittest.TestCase):
maxDiff = None
def setUp(self):
self.operation = mock.Mock()
self.paginate_config = {
'output_token': 'NextToken',
'py_input_token': 'NextToken',
'result_key': 'ResultKey',
'non_aggregate_keys': ['NotResultKey'],
}
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
def set_responses(self, responses):
complete_responses = []
for response in responses:
complete_responses.append((None, response))
self.operation.call.side_effect = complete_responses
def test_include_non_aggregate_keys(self):
self.set_responses([
{'ResultKey': ['foo'], 'NotResultKey': 'a', 'NextToken': 't1'},
{'ResultKey': ['bar'], 'NotResultKey': 'a', 'NextToken': 't2'},
{'ResultKey': ['baz'], 'NotResultKey': 'a'},
])
pages = self.paginator.paginate(None)
actual = pages.build_full_result()
self.assertEqual(pages.non_aggregate_part, {'NotResultKey': 'a'})
expected = {
'ResultKey': ['foo', 'bar', 'baz'],
'NotResultKey': 'a',
}
self.assertEqual(actual, expected)
def test_include_with_multiple_result_keys(self):
self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2']
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
self.set_responses([
{'ResultKey1': ['a', 'b'], 'ResultKey2': ['u', 'v'],
'NotResultKey': 'a', 'NextToken': 'token1'},
{'ResultKey1': ['c', 'd'], 'ResultKey2': ['w', 'x'],
'NotResultKey': 'a', 'NextToken': 'token2'},
{'ResultKey1': ['e', 'f'], 'ResultKey2': ['y', 'z'],
'NotResultKey': 'a',}
])
pages = self.paginator.paginate(None)
actual = pages.build_full_result()
expected = {
'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'],
'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'],
'NotResultKey': 'a',
}
self.assertEqual(actual, expected)
def test_include_with_nested_result_keys(self):
self.paginate_config['result_key'] = 'Result.Key'
self.paginate_config['non_aggregate_keys'] = [
'Outer', 'Result.Inner',
]
self.operation.pagination = self.paginate_config
self.paginator = Paginator(self.operation)
self.set_responses([
# The non result keys shows hypothetical
# example. This doesn't actually happen,
# but in the case where the non result keys
# are different across pages, we use the values
# from the first page.
{'Result': {'Key': ['foo'], 'Inner': 'v1'},
'Outer': 'v2', 'NextToken': 't1'},
{'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'},
'Outer': 'v4', 'NextToken': 't2'},
{'Result': {'Key': ['qux'], 'Inner': 'v5'},
'Outer': 'v6', 'NextToken': 't3'},
])
pages = self.paginator.paginate(None)
actual = pages.build_full_result()
self.assertEqual(pages.non_aggregate_part,
{'Outer': 'v2', 'Result': {'Inner': 'v1'}})
expected = {
'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'},
'Outer': 'v2',
}
self.assertEqual(actual, expected)
class TestMultipleInputKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_build_full_result_with_multiple_input_keys(self):
responses = [{
"Users": ["User1", "User2"],
"Groups": ["Group1"],
"Marker1": "m1",
"Marker2": "m2"
}, {
"Users": ["User3", "User4"],
"Groups": ["Group2"],
"Marker1": "m3",
"Marker2": "m4"
}, {
"Users": ["User5"],
"Groups": ["Group3"]
}]
self.method.side_effect = responses
pages = self.paginator.paginate(PaginationConfig={'MaxItems': 3})
complete = pages.build_full_result()
self.assertEqual(
complete, {
"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2'],
"NextToken": "m1___m2___1"
})
def test_resume_with_multiple_input_keys(self):
responses = [
{
"Users": ["User3", "User4"],
"Groups": ["Group2"],
"Marker1": "m3",
"Marker2": "m4"
},
{
"Users": ["User5"],
"Groups": ["Group3"]
},
]
self.method.side_effect = responses
pages = self.paginator.paginate(PaginationConfig={
'MaxItems': 1,
'StartingToken': 'm1___m2___1'
})
complete = pages.build_full_result()
self.assertEqual(complete, {
"Users": ['User4'],
"Groups": [],
"NextToken": "m3___m4"
})
self.assertEqual(self.method.call_args_list,
[mock.call(InMarker1='m1', InMarker2='m2')])
def test_result_key_exposed_on_paginator(self):
self.assertEqual([rk.expression for rk in self.paginator.result_keys],
['Users', 'Groups'])
def test_result_key_exposed_on_page_iterator(self):
pages = self.paginator.paginate(MaxItems=3)
self.assertEqual([rk.expression for rk in pages.result_keys],
['Users', 'Groups'])
def paginate(self, **kwargs):
return Paginator.paginate(self, **kwargs)
def paginate(self, **kwargs):
return Paginator.paginate(self, **kwargs)
class TestMultipleInputKeys(unittest.TestCase):
def setUp(self):
self.method = mock.Mock()
# Probably the most complicated example we'll see:
# multiple input/output/result keys.
self.paginate_config = {
"output_token": ["Marker1", "Marker2"],
"input_token": ["InMarker1", "InMarker2"],
"result_key": ["Users", "Groups"],
}
self.paginator = Paginator(self.method, self.paginate_config)
def test_build_full_result_with_multiple_input_keys(self):
responses = [{
"Users": ["User1", "User2"],
"Groups": ["Group1"],
"Marker1": "m1",
"Marker2": "m2"
}, {
"Users": ["User3", "User4"],
"Groups": ["Group2"],
"Marker1": "m3",
"Marker2": "m4"
}, {
"Users": ["User5"],
"Groups": ["Group3"]
}]
self.method.side_effect = responses
pages = self.paginator.paginate(PaginationConfig={'MaxItems': 3})
complete = pages.build_full_result()
expected_token = encode_token({
"InMarker1": "m1",
"InMarker2": "m2",
"boto_truncate_amount": 1
})
self.assertEqual(
complete, {
"Users": ['User1', 'User2', 'User3'],
"Groups": ['Group1', 'Group2'],
"NextToken": expected_token
})
def test_resume_with_multiple_input_keys(self):
responses = [
{
"Users": ["User3", "User4"],
"Groups": ["Group2"],
"Marker1": "m3",
"Marker2": "m4"
},
{
"Users": ["User5"],
"Groups": ["Group3"]
},
]
self.method.side_effect = responses
starting_token = encode_token({
"InMarker1": "m1",
"InMarker2": "m2",
"boto_truncate_amount": 1
})
pages = self.paginator.paginate(PaginationConfig={
'MaxItems': 1,
'StartingToken': starting_token
})
complete = pages.build_full_result()
expected_token = encode_token({"InMarker1": "m3", "InMarker2": "m4"})
self.assertEqual(complete, {
"Users": ['User4'],
"Groups": [],
"NextToken": expected_token
})
self.assertEqual(self.method.call_args_list,
[mock.call(InMarker1='m1', InMarker2='m2')])
def test_resume_encounters_an_empty_payload(self):
response = {"not_a_result_key": "it happens in some service"}
self.method.return_value = response
starting_token = encode_token({
"Marker": None,
"boto_truncate_amount": 1
})
complete = self.paginator \
.paginate(PaginationConfig={'StartingToken': starting_token}) \
.build_full_result()
self.assertEqual(complete, {})
def test_result_key_exposed_on_paginator(self):
self.assertEqual([rk.expression for rk in self.paginator.result_keys],
['Users', 'Groups'])
def test_result_key_exposed_on_page_iterator(self):
pages = self.paginator.paginate(MaxItems=3)
self.assertEqual([rk.expression for rk in pages.result_keys],
['Users', 'Groups'])