class TestSearchOverResults(unittest.TestCase): def setUp(self): self.method = mock.Mock() self.paginate_config = { 'more_results': 'IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.method, self.paginate_config) responses = [ {'Foo': [{'a': 1}, {'b': 2}], 'IsTruncated': True, 'NextToken': '1'}, {'Foo': [{'a': 3}, {'b': 4}], 'IsTruncated': True, 'NextToken': '2'}, {'Foo': [{'a': 5}], 'IsTruncated': False, 'NextToken': '3'} ] self.method.side_effect = responses def test_yields_non_list_values(self): result = list(self.paginator.paginate().search('Foo[0].a')) self.assertEqual([1, 3, 5], result) def test_yields_individual_list_values(self): result = list(self.paginator.paginate().search('Foo[].*[]')) self.assertEqual([1, 2, 3, 4, 5], result) def test_empty_when_no_match(self): result = list(self.paginator.paginate().search('Foo[].qux')) self.assertEqual([], result) def test_no_yield_when_no_match_on_page(self): result = list(self.paginator.paginate().search('Foo[].b')) self.assertEqual([2, 4], result)
class TestIncludeResultKeys(unittest.TestCase): def setUp(self): self.method = mock.Mock() self.paginate_config = { 'output_token': 'Marker', 'input_token': 'Marker', 'result_key': ['ResultKey', 'Count', 'Log'], } self.paginator = Paginator(self.method, self.paginate_config) def test_different_kinds_of_result_key(self): self.method.side_effect = [ {'ResultKey': ['a'], 'Count': 1, 'Log': 'x', 'Marker': 'a'}, {'not_a_result_key': 'this page will be ignored', 'Marker': '_'}, {'ResultKey': ['b', 'c'], 'Count': 2, 'Log': 'y', 'Marker': 'b'}, {'ResultKey': ['d', 'e', 'f'], 'Count': 3, 'Log': 'z'}, ] pages = self.paginator.paginate() expected = { 'ResultKey': ['a', 'b', 'c', 'd', 'e', 'f'], 'Count': 6, 'Log': 'xyz', } self.assertEqual(pages.build_full_result(), expected) def test_result_key_is_missing(self): self.method.side_effect = [ {'not_a_result_key': 'this page will be ignored', 'Marker': '_'}, {'neither_this_one': 'this page will be ignored, too'}, ] pages = self.paginator.paginate() expected = {} self.assertEqual(pages.build_full_result(), expected)
class TestMultipleTokens(unittest.TestCase): def setUp(self): self.method = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": ["ListBucketResults.NextKeyMarker", "ListBucketResults.NextUploadIdMarker"], "input_token": ["key_marker", "upload_id_marker"], "result_key": 'Foo', } self.paginator = Paginator(self.method, self.paginate_config) def test_s3_list_multipart_uploads(self): responses = [ {"Foo": [1], "ListBucketResults": {"NextKeyMarker": "key1", "NextUploadIdMarker": "up1"}}, {"Foo": [2], "ListBucketResults": {"NextKeyMarker": "key2", "NextUploadIdMarker": "up2"}}, {"Foo": [3], "ListBucketResults": {"NextKeyMarker": "key3", "NextUploadIdMarker": "up3"}}, {} ] self.method.side_effect = responses list(self.paginator.paginate()) self.assertEqual( self.method.call_args_list, [mock.call(), mock.call(key_marker='key1', upload_id_marker='up1'), mock.call(key_marker='key2', upload_id_marker='up2'), mock.call(key_marker='key3', upload_id_marker='up3'), ])
def test_next_token_on_page_boundary(self): paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses expected_token = encode_token({"Marker": "m2"}) self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 2}).build_full_result(), {'Users': ['User1', 'User2'], 'NextToken': expected_token})
def test_max_items_can_be_specified(self): paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses expected_token = encode_token({"Marker": "m1"}) self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 1}).build_full_result(), {'Users': ['User1'], 'NextToken': expected_token})
class TestPaginatorWithPathExpressions(unittest.TestCase): def setUp(self): self.method = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { 'output_token': [ 'NextMarker || ListBucketResult.Contents[-1].Key'], 'input_token': 'next_marker', 'result_key': 'Contents', } self.paginator = Paginator(self.method, self.paginate_config) def test_s3_list_objects(self): responses = [ {'NextMarker': 'token1'}, {'NextMarker': 'token2'}, {'not_next_token': 'foo'}] self.method.side_effect = responses list(self.paginator.paginate()) self.assertEqual( self.method.call_args_list, [mock.call(), mock.call(next_marker='token1'), mock.call(next_marker='token2')]) def test_s3_list_object_complex(self): responses = [ {'NextMarker': 'token1'}, {'ListBucketResult': { 'Contents': [{"Key": "first"}, {"Key": "Last"}]}}, {'not_next_token': 'foo'}] self.method.side_effect = responses list(self.paginator.paginate()) self.assertEqual( self.method.call_args_list, [mock.call(), mock.call(next_marker='token1'), mock.call(next_marker='Last')])
def test_max_items_exceeds_actual_amount(self): # Because MaxItems=10 > number of users (3), we should just return # all of the users. paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 10}).build_full_result(), {'Users': ['User1', 'User2', 'User3']})
class TestPaginatorPageSize(unittest.TestCase): def setUp(self): self.method = mock.Mock() self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": ["Users", "Groups"], 'limit_key': 'MaxKeys', } self.paginator = Paginator(self.method, self.paginate_config) self.endpoint = mock.Mock() def test_no_page_size(self): kwargs = {'arg1': 'foo', 'arg2': 'bar'} ref_kwargs = {'arg1': 'foo', 'arg2': 'bar'} pages = self.paginator.paginate(**kwargs) pages._inject_starting_params(kwargs) self.assertEqual(kwargs, ref_kwargs) def test_page_size(self): kwargs = {'arg1': 'foo', 'arg2': 'bar', 'PaginationConfig': {'PageSize': 5}} extracted_kwargs = {'arg1': 'foo', 'arg2': 'bar'} # Note that ``MaxKeys`` in ``setUp()`` is the parameter used for # the page size for pagination. ref_kwargs = {'arg1': 'foo', 'arg2': 'bar', 'MaxKeys': 5} pages = self.paginator.paginate(**kwargs) pages._inject_starting_params(extracted_kwargs) self.assertEqual(extracted_kwargs, ref_kwargs) def test_page_size_incorrectly_provided(self): kwargs = {'arg1': 'foo', 'arg2': 'bar', 'PaginationConfig': {'PageSize': 5}} del self.paginate_config['limit_key'] with self.assertRaises(PaginationError): self.paginator.paginate(**kwargs)
class TestExpressionKeyIterators(unittest.TestCase): def setUp(self): self.method = mock.Mock() # This is something like what we'd see in RDS. self.paginate_config = { "input_token": "Marker", "output_token": "Marker", "limit_key": "MaxRecords", "result_key": "EngineDefaults.Parameters" } self.paginator = Paginator(self.method, self.paginate_config) self.responses = [ {"EngineDefaults": {"Parameters": ["One", "Two"]}, "Marker": "m1"}, {"EngineDefaults": {"Parameters": ["Three", "Four"]}, "Marker": "m2"}, {"EngineDefaults": {"Parameters": ["Five"]}} ] def test_result_key_iters(self): self.method.side_effect = self.responses pages = self.paginator.paginate() iterators = pages.result_key_iters() self.assertEqual(len(iterators), 1) self.assertEqual(list(iterators[0]), ['One', 'Two', 'Three', 'Four', 'Five']) def test_build_full_result_with_single_key(self): self.method.side_effect = self.responses pages = self.paginator.paginate() complete = pages.build_full_result() self.assertEqual(complete, { 'EngineDefaults': { 'Parameters': ['One', 'Two', 'Three', 'Four', 'Five'] }, })
def test_max_items_can_be_specified_truncates_response(self): # We're saying we only want 4 items, but notice that the second # page of results returns users 4-6 so we have to truncated # part of that second page. paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1", "User2", "User3"], "Marker": "m1"}, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}, {"Users": ["User7"]}, ] self.method.side_effect = responses expected_token = encode_token( {"Marker": "m1", "boto_truncate_amount": 1}) self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 4}).build_full_result(), {'Users': ['User1', 'User2', 'User3', 'User4'], 'NextToken': expected_token})
class TestOptionalTokens(unittest.TestCase): """ Tests a paginator with an optional output token. If this gets left in the request params from a previous page, the API will skip over a record. """ def setUp(self): self.method = mock.Mock() self.paginate_config = { "output_token": ["NextRecordName", "NextRecordType", "NextRecordIdentifier"], "input_token": ["StartRecordName", "StartRecordType", "StartRecordIdentifier"], "result_key": 'Foo', } self.paginator = Paginator(self.method, self.paginate_config) def test_clean_token(self): responses = [ {"Foo": [1], "IsTruncated": True, "NextRecordName": "aaa.example.com", "NextRecordType": "A", "NextRecordIdentifier": "id"}, {"Foo": [2], "IsTruncated": True, "NextRecordName": "bbb.example.com", "NextRecordType": "A"}, {"Foo": [3], "IsTruncated": False}, ] self.method.side_effect = responses list(self.paginator.paginate()) self.assertEqual( self.method.call_args_list, [mock.call(), mock.call(StartRecordName='aaa.example.com', StartRecordType='A', StartRecordIdentifier='id'), mock.call(StartRecordName='bbb.example.com', StartRecordType='A') ])
def test_resume_next_marker_mid_page(self): # This is a simulation of picking up from the response # from test_MaxItems_can_be_specified_truncates_response # We got the first 4 users, when we pick up we should get # User5 - User7. paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User4", "User5", "User6"], "Marker": "m2"}, {"Users": ["User7"]}, ] self.method.side_effect = responses starting_token = encode_token( {"Marker": "m1", "boto_truncate_amount": 1}) pagination_config = {'StartingToken': starting_token} self.assertEqual( paginator.paginate( PaginationConfig=pagination_config).build_full_result(), {'Users': ['User5', 'User6', 'User7']}) self.assertEqual( self.method.call_args_list, [mock.call(Marker='m1'), mock.call(Marker='m2')])
class TestIncludeNonResultKeys(unittest.TestCase): maxDiff = None def setUp(self): self.method = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'ResultKey', 'non_aggregate_keys': ['NotResultKey'], } self.paginator = Paginator(self.method, self.paginate_config) def test_include_non_aggregate_keys(self): self.method.side_effect = [ {'ResultKey': ['foo'], 'NotResultKey': 'a', 'NextToken': 't1'}, {'ResultKey': ['bar'], 'NotResultKey': 'a', 'NextToken': 't2'}, {'ResultKey': ['baz'], 'NotResultKey': 'a'}, ] pages = self.paginator.paginate() actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, {'NotResultKey': 'a'}) expected = { 'ResultKey': ['foo', 'bar', 'baz'], 'NotResultKey': 'a', } self.assertEqual(actual, expected) def test_include_with_multiple_result_keys(self): self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2'] self.paginator = Paginator(self.method, self.paginate_config) self.method.side_effect = [ {'ResultKey1': ['a', 'b'], 'ResultKey2': ['u', 'v'], 'NotResultKey': 'a', 'NextToken': 'token1'}, {'ResultKey1': ['c', 'd'], 'ResultKey2': ['w', 'x'], 'NotResultKey': 'a', 'NextToken': 'token2'}, {'ResultKey1': ['e', 'f'], 'ResultKey2': ['y', 'z'], 'NotResultKey': 'a'} ] pages = self.paginator.paginate() actual = pages.build_full_result() expected = { 'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'], 'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'], 'NotResultKey': 'a', } self.assertEqual(actual, expected) def test_include_with_nested_result_keys(self): self.paginate_config['result_key'] = 'Result.Key' self.paginate_config['non_aggregate_keys'] = [ 'Outer', 'Result.Inner', ] self.paginator = Paginator(self.method, self.paginate_config) self.method.side_effect = [ # The non result keys shows hypothetical # example. This doesn't actually happen, # but in the case where the non result keys # are different across pages, we use the values # from the first page. {'Result': {'Key': ['foo'], 'Inner': 'v1'}, 'Outer': 'v2', 'NextToken': 't1'}, {'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'}, 'Outer': 'v4', 'NextToken': 't2'}, {'Result': {'Key': ['qux'], 'Inner': 'v5'}, 'Outer': 'v6', 'NextToken': 't3'}, ] pages = self.paginator.paginate() actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, {'Outer': 'v2', 'Result': {'Inner': 'v1'}}) expected = { 'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'}, 'Outer': 'v2', } self.assertEqual(actual, expected)
class TestMultipleInputKeys(unittest.TestCase): def setUp(self): self.method = mock.Mock() # Probably the most complicated example we'll see: # multiple input/output/result keys. self.paginate_config = { "output_token": ["Marker1", "Marker2"], "input_token": ["InMarker1", "InMarker2"], "result_key": ["Users", "Groups"], } self.paginator = Paginator(self.method, self.paginate_config) def test_build_full_result_with_multiple_input_keys(self): responses = [ {"Users": ["User1", "User2"], "Groups": ["Group1"], "Marker1": "m1", "Marker2": "m2"}, {"Users": ["User3", "User4"], "Groups": ["Group2"], "Marker1": "m3", "Marker2": "m4"}, {"Users": ["User5"], "Groups": ["Group3"]} ] self.method.side_effect = responses pages = self.paginator.paginate( PaginationConfig={'MaxItems': 3}) complete = pages.build_full_result() expected_token = encode_token( {"InMarker1": "m1", "InMarker2": "m2", "boto_truncate_amount": 1}) self.assertEqual(complete, {"Users": ['User1', 'User2', 'User3'], "Groups": ['Group1', 'Group2'], "NextToken": expected_token}) def test_resume_with_multiple_input_keys(self): responses = [ {"Users": ["User3", "User4"], "Groups": ["Group2"], "Marker1": "m3", "Marker2": "m4"}, {"Users": ["User5"], "Groups": ["Group3"]}, ] self.method.side_effect = responses starting_token = encode_token( {"InMarker1": "m1", "InMarker2": "m2", "boto_truncate_amount": 1}) pages = self.paginator.paginate( PaginationConfig={'MaxItems': 1, 'StartingToken': starting_token}) complete = pages.build_full_result() expected_token = encode_token( {"InMarker1": "m3", "InMarker2": "m4"}) self.assertEqual(complete, {"Users": ['User4'], "Groups": [], "NextToken": expected_token}) self.assertEqual( self.method.call_args_list, [mock.call(InMarker1='m1', InMarker2='m2')]) def test_resume_encounters_an_empty_payload(self): response = {"not_a_result_key": "it happens in some service"} self.method.return_value = response starting_token = encode_token( {"Marker": None, "boto_truncate_amount": 1}) complete = self.paginator \ .paginate(PaginationConfig={'StartingToken': starting_token}) \ .build_full_result() self.assertEqual(complete, {}) def test_result_key_exposed_on_paginator(self): self.assertEqual( [rk.expression for rk in self.paginator.result_keys], ['Users', 'Groups'] ) def test_result_key_exposed_on_page_iterator(self): pages = self.paginator.paginate(MaxItems=3) self.assertEqual( [rk.expression for rk in pages.result_keys], ['Users', 'Groups'] )
class TestMultipleResultKeys(unittest.TestCase): def setUp(self): self.method = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": ["Users", "Groups"], } self.paginator = Paginator(self.method, self.paginate_config) def test_build_full_result_with_multiple_result_keys(self): responses = [ {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}, {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"}, {"Users": ["User3"], "Groups": ["Group3"]}, ] self.method.side_effect = responses pages = self.paginator.paginate() complete = pages.build_full_result() self.assertEqual(complete, {"Users": ['User1', 'User2', 'User3'], "Groups": ['Group1', 'Group2', 'Group3']}) def test_build_full_result_with_different_length_result_keys(self): responses = [ {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}, # Then we stop getting "Users" output, but we get more "Groups" {"Users": [], "Groups": ["Group2"], "Marker": "m2"}, {"Users": [], "Groups": ["Group3"]}, ] self.method.side_effect = responses pages = self.paginator.paginate() complete = pages.build_full_result() self.assertEqual(complete, {"Users": ['User1'], "Groups": ['Group1', 'Group2', 'Group3']}) def test_build_full_result_with_zero_length_result_key(self): responses = [ # In this case the 'Users' key is always empty but we should # have a 'Users' key in the output, it should just have an # empty list for a value. {"Users": [], "Groups": ["Group1"], "Marker": "m1"}, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}, {"Users": [], "Groups": ["Group3"]}, ] self.method.side_effect = responses pages = self.paginator.paginate() complete = pages.build_full_result() self.assertEqual(complete, {"Users": [], "Groups": ['Group1', 'Group2', 'Group3']}) def test_build_result_with_secondary_keys(self): responses = [ {"Users": ["User1", "User2"], "Groups": ["Group1", "Group2"], "Marker": "m1"}, {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"}, {"Users": ["User4"], "Groups": ["Group4"]}, ] self.method.side_effect = responses pages = self.paginator.paginate( PaginationConfig={'MaxItems': 1}) complete = pages.build_full_result() expected_token = encode_token( {"Marker": None, "boto_truncate_amount": 1}) self.assertEqual(complete, {"Users": ["User1"], "Groups": ["Group1", "Group2"], "NextToken": expected_token}) def test_resume_with_secondary_keys(self): # This is simulating a continutation of the previous test, # test_build_result_with_secondary_keys. We use the # token specified in the response "None___1" to continue where we # left off. responses = [ {"Users": ["User1", "User2"], "Groups": ["Group1", "Group2"], "Marker": "m1"}, {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"}, {"Users": ["User4"], "Groups": ["Group4"]}, ] self.method.side_effect = responses starting_token = encode_token( {"Marker": None, "boto_truncate_amount": 1}) pages = self.paginator.paginate( PaginationConfig={'MaxItems': 1, 'StartingToken': starting_token}) complete = pages.build_full_result() # Note that the secondary keys ("Groups") are all truncated because # they were in the original (first) response. expected_token = encode_token({"Marker": "m1"}) self.assertEqual(complete, {"Users": ["User2"], "Groups": [], "NextToken": expected_token}) def test_resume_with_secondary_result_as_string(self): self.method.return_value = {"Users": ["User1", "User2"], "Groups": "a"} starting_token = encode_token( {"Marker": None, "boto_truncate_amount": 1}) pages = self.paginator.paginate( PaginationConfig={'MaxItems': 1, 'StartingToken': starting_token}) complete = pages.build_full_result() # Note that the secondary keys ("Groups") becomes empty string because # they were in the original (first) response. self.assertEqual(complete, {"Users": ["User2"], "Groups": ""}) def test_resume_with_secondary_result_as_integer(self): self.method.return_value = {"Users": ["User1", "User2"], "Groups": 123} starting_token = encode_token( {"Marker": None, "boto_truncate_amount": 1}) pages = self.paginator.paginate( PaginationConfig={'MaxItems': 1, 'StartingToken': starting_token}) complete = pages.build_full_result() # Note that the secondary keys ("Groups") becomes zero because # they were in the original (first) response. self.assertEqual(complete, {"Users": ["User2"], "Groups": 0})
class TestKeyIterators(unittest.TestCase): def setUp(self): self.method = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": "Users" } self.paginator = Paginator(self.method, self.paginate_config) def test_result_key_iters(self): responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses pages = self.paginator.paginate() iterators = pages.result_key_iters() self.assertEqual(len(iterators), 1) self.assertEqual(list(iterators[0]), ["User1", "User2", "User3"]) def test_build_full_result_with_single_key(self): responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses pages = self.paginator.paginate() complete = pages.build_full_result() self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']}) def test_max_items_can_be_specified(self): paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses expected_token = encode_token({"Marker": "m1"}) self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 1}).build_full_result(), {'Users': ['User1'], 'NextToken': expected_token}) def test_max_items_as_strings(self): paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses expected_token = encode_token({"Marker": "m1"}) self.assertEqual( # Note MaxItems is a string here. paginator.paginate( PaginationConfig={'MaxItems': '1'}).build_full_result(), {'Users': ['User1'], 'NextToken': expected_token}) def test_next_token_on_page_boundary(self): paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses expected_token = encode_token({"Marker": "m2"}) self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 2}).build_full_result(), {'Users': ['User1', 'User2'], 'NextToken': expected_token}) def test_max_items_can_be_specified_truncates_response(self): # We're saying we only want 4 items, but notice that the second # page of results returns users 4-6 so we have to truncated # part of that second page. paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1", "User2", "User3"], "Marker": "m1"}, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}, {"Users": ["User7"]}, ] self.method.side_effect = responses expected_token = encode_token( {"Marker": "m1", "boto_truncate_amount": 1}) self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 4}).build_full_result(), {'Users': ['User1', 'User2', 'User3', 'User4'], 'NextToken': expected_token}) def test_resume_next_marker_mid_page(self): # This is a simulation of picking up from the response # from test_MaxItems_can_be_specified_truncates_response # We got the first 4 users, when we pick up we should get # User5 - User7. paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User4", "User5", "User6"], "Marker": "m2"}, {"Users": ["User7"]}, ] self.method.side_effect = responses starting_token = encode_token( {"Marker": "m1", "boto_truncate_amount": 1}) pagination_config = {'StartingToken': starting_token} self.assertEqual( paginator.paginate( PaginationConfig=pagination_config).build_full_result(), {'Users': ['User5', 'User6', 'User7']}) self.assertEqual( self.method.call_args_list, [mock.call(Marker='m1'), mock.call(Marker='m2')]) def test_max_items_exceeds_actual_amount(self): # Because MaxItems=10 > number of users (3), we should just return # all of the users. paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses self.assertEqual( paginator.paginate( PaginationConfig={'MaxItems': 10}).build_full_result(), {'Users': ['User1', 'User2', 'User3']}) def test_bad_input_tokens(self): responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses with self.assertRaisesRegexp(ValueError, 'Bad starting token'): pagination_config = {'StartingToken': 'does___not___work'} self.paginator.paginate( PaginationConfig=pagination_config).build_full_result()
def paginate(self, **kwargs): return Paginator.paginate(self, **kwargs)
class TestPagination(unittest.TestCase): def setUp(self): self.method = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.method, self.paginate_config) def test_result_key_available(self): self.assertEqual( [rk.expression for rk in self.paginator.result_keys], ['Foo'] ) def test_no_next_token(self): response = {'not_the_next_token': 'foobar'} self.method.return_value = response actual = list(self.paginator.paginate()) self.assertEqual(actual, [{'not_the_next_token': 'foobar'}]) def test_next_token_in_response(self): responses = [{'NextToken': 'token1'}, {'NextToken': 'token2'}, {'not_next_token': 'foo'}] self.method.side_effect = responses actual = list(self.paginator.paginate()) self.assertEqual(actual, responses) # The first call has no next token, the second and third call should # have 'token1' and 'token2' respectively. self.assertEqual(self.method.call_args_list, [mock.call(), mock.call(NextToken='token1'), mock.call(NextToken='token2')]) def test_next_token_is_string(self): self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": "Users", "limit_key": "MaxKeys", } self.paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]} ] self.method.side_effect = responses result = self.paginator.paginate(PaginationConfig={'MaxItems': 1}) result = result.build_full_result() token = result.get('NextToken') self.assertIsInstance(token, six.string_types) def test_any_passed_in_args_are_unmodified(self): responses = [{'NextToken': 'token1'}, {'NextToken': 'token2'}, {'not_next_token': 'foo'}] self.method.side_effect = responses actual = list(self.paginator.paginate(Foo='foo', Bar='bar')) self.assertEqual(actual, responses) self.assertEqual( self.method.call_args_list, [mock.call(Foo='foo', Bar='bar'), mock.call(Foo='foo', Bar='bar', NextToken='token1'), mock.call(Foo='foo', Bar='bar', NextToken='token2')]) def test_exception_raised_if_same_next_token(self): responses = [{'NextToken': 'token1'}, {'NextToken': 'token2'}, {'NextToken': 'token2'}] self.method.side_effect = responses with self.assertRaises(PaginationError): list(self.paginator.paginate()) def test_next_token_with_or_expression(self): self.pagination_config = { 'output_token': 'NextToken || NextToken2', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.method, self.pagination_config) # Verify that despite varying between NextToken and NextToken2 # we still can extract the right next tokens. responses = [ {'NextToken': 'token1'}, {'NextToken2': 'token2'}, # The first match found wins, so because NextToken is # listed before NextToken2 in the 'output_tokens' config, # 'token3' is chosen over 'token4'. {'NextToken': 'token3', 'NextToken2': 'token4'}, {'not_next_token': 'foo'}, ] self.method.side_effect = responses list(self.paginator.paginate()) self.assertEqual( self.method.call_args_list, [mock.call(), mock.call(NextToken='token1'), mock.call(NextToken='token2'), mock.call(NextToken='token3')]) def test_more_tokens(self): # Some pagination configs have a 'more_token' key that # indicate whether or not the results are being paginated. self.paginate_config = { 'more_results': 'IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.method, self.paginate_config) responses = [ {'Foo': [1], 'IsTruncated': True, 'NextToken': 'token1'}, {'Foo': [2], 'IsTruncated': True, 'NextToken': 'token2'}, {'Foo': [3], 'IsTruncated': False, 'NextToken': 'token3'}, {'Foo': [4], 'not_next_token': 'foo'}, ] self.method.side_effect = responses list(self.paginator.paginate()) self.assertEqual( self.method.call_args_list, [mock.call(), mock.call(NextToken='token1'), mock.call(NextToken='token2')]) def test_more_tokens_is_path_expression(self): self.paginate_config = { 'more_results': 'Foo.IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Bar', } self.paginator = Paginator(self.method, self.paginate_config) responses = [ {'Foo': {'IsTruncated': True}, 'NextToken': 'token1'}, {'Foo': {'IsTruncated': False}, 'NextToken': 'token2'}, ] self.method.side_effect = responses list(self.paginator.paginate()) self.assertEqual( self.method.call_args_list, [mock.call(), mock.call(NextToken='token1')]) def test_page_size(self): self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": "Users", "limit_key": "MaxKeys", } self.paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]}, ] self.method.side_effect = responses users = [] for page in self.paginator.paginate(PaginationConfig={'PageSize': 1}): users += page['Users'] self.assertEqual( self.method.call_args_list, [mock.call(MaxKeys=1), mock.call(Marker='m1', MaxKeys=1), mock.call(Marker='m2', MaxKeys=1)] ) def test_with_empty_markers(self): responses = [ {"Users": ["User1"], "Marker": ""}, {"Users": ["User1"], "Marker": ""}, {"Users": ["User1"], "Marker": ""} ] self.method.side_effect = responses users = [] for page in self.paginator.paginate(): users += page['Users'] # We want to stop paginating if the next token is empty. self.assertEqual( self.method.call_args_list, [mock.call()] ) self.assertEqual(users, ['User1']) def test_build_full_result_with_single_key(self): self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": "Users", "limit_key": "MaxKeys", } self.paginator = Paginator(self.method, self.paginate_config) responses = [ {"Users": ["User1"], "Marker": "m1"}, {"Users": ["User2"], "Marker": "m2"}, {"Users": ["User3"]} ] self.method.side_effect = responses pages = self.paginator.paginate() complete = pages.build_full_result() self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})