class TestPaginatorPageSize(unittest.TestCase): def setUp(self): self.operation = mock.Mock() self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": ["Users", "Groups"], 'limit_key': 'MaxKeys', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.endpoint = mock.Mock() def test_no_page_size(self): kwargs = {'arg1': 'foo', 'arg2': 'bar'} ref_kwargs = {'arg1': 'foo', 'arg2': 'bar'} pages = self.paginator.paginate(self.endpoint, **kwargs) pages._inject_starting_params(kwargs) self.assertEqual(kwargs, ref_kwargs) def test_page_size(self): kwargs = {'arg1': 'foo', 'arg2': 'bar', 'page_size': 5} extracted_kwargs = {'arg1': 'foo', 'arg2': 'bar'} # Note that ``MaxKeys`` in ``setUp()`` is the parameter used for # the page size for pagination. ref_kwargs = {'arg1': 'foo', 'arg2': 'bar', 'MaxKeys': 5} pages = self.paginator.paginate(self.endpoint, **kwargs) pages._inject_starting_params(extracted_kwargs) self.assertEqual(extracted_kwargs, ref_kwargs)
def test_max_items_as_strings(self): # Some services (route53) model MaxItems as a string type. # We need to be able to handle this case. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( # Note max_items is a string here. paginator.paginate(None, max_items='1').build_full_result(), { 'Users': ['User1'], 'NextToken': 'm1' })
class TestMultipleInputKeys(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # Probably the most complicated example we'll see: # multiple input/output/result keys. self.paginate_config = { "output_token": ["Marker1", "Marker2"], "input_token": ["InMarker1", "InMarker2"], "result_key": ["Users", "Groups"], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_build_full_result_with_multiple_input_keys(self): responses = [ (None, {"Users": ["User1", "User2"], "Groups": ["Group1"], "Marker1": "m1", "Marker2": "m2"}), (None, {"Users": ["User3", "User4"], "Groups": ["Group2"], "Marker1": "m3", "Marker2": "m4"}), (None, {"Users": ["User5"], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=3) complete = pages.build_full_result() self.assertEqual(complete, {"Users": ['User1', 'User2', 'User3'], "Groups": ['Group1', 'Group2'], "NextToken": "m1___m2___1"}) def test_resume_with_multiple_input_keys(self): responses = [ (None, {"Users": ["User3", "User4"], "Groups": ["Group2"], "Marker1": "m3", "Marker2": "m4"}), (None, {"Users": ["User5"], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=1, starting_token='m1___m2___1') complete = pages.build_full_result() self.assertEqual(complete, {"Users": ['User4'], "Groups": [], "NextToken": "m3___m4"}) self.assertEqual( self.operation.call.call_args_list, [mock.call(None, InMarker1='m1', InMarker2='m2'),]) def test_result_key_exposed_on_paginator(self): self.assertEqual( [rk.expression for rk in self.paginator.result_keys], ['Users', 'Groups'] ) def test_result_key_exposed_on_page_iterator(self): pages = self.paginator.paginate(None, max_items=3) self.assertEqual( [rk.expression for rk in pages.result_keys], ['Users', 'Groups'] )
def test_max_items_can_be_specified(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=1).build_full_result(), {'Users': ['User1'], 'NextToken': 'm1'})
def test_next_token_on_page_boundary(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=2).build_full_result(), {'Users': ['User1', 'User2'], 'NextToken': 'm2'})
def test_max_items_exceeds_actual_amount(self): # Because MaxItems=10 > number of users (3), we should just return # all of the users. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=10).build_full_result(), {'Users': ['User1', 'User2', 'User3']})
class TestPaginatorWithPathExpressions(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { 'output_token': ['NextMarker || ListBucketResult.Contents[-1].Key'], 'input_token': 'next_marker', 'result_key': 'Contents', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_s3_list_objects(self): responses = [(None, { 'NextMarker': 'token1' }), (None, { 'NextMarker': 'token2' }), (None, { 'not_next_token': 'foo' })] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, next_marker='token1'), mock.call(None, next_marker='token2'), ]) def test_s3_list_object_complex(self): responses = [(None, { 'NextMarker': 'token1' }), (None, { 'ListBucketResult': { 'Contents': [{ "Key": "first" }, { "Key": "Last" }] } }), (None, { 'not_next_token': 'foo' })] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, next_marker='token1'), mock.call(None, next_marker='Last'), ])
class TestExpressionKeyIterators(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something like what we'd see in RDS. self.paginate_config = { "input_token": "Marker", "output_token": "Marker", "limit_key": "MaxRecords", "result_key": "EngineDefaults.Parameters" } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.responses = [ (None, { "EngineDefaults": { "Parameters": ["One", "Two"] }, "Marker": "m1" }), (None, { "EngineDefaults": { "Parameters": ["Three", "Four"] }, "Marker": "m2" }), (None, { "EngineDefaults": { "Parameters": ["Five"] } }), ] def test_result_key_iters(self): self.operation.call.side_effect = self.responses pages = self.paginator.paginate(None) iterators = pages.result_key_iters() self.assertEqual(len(iterators), 1) self.assertEqual(list(iterators[0]), ['One', 'Two', 'Three', 'Four', 'Five']) def test_build_full_result_with_single_key(self): self.operation.call.side_effect = self.responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual( complete, { 'EngineDefaults': { 'Parameters': ['One', 'Two', 'Three', 'Four', 'Five'] }, })
def test_max_items_as_strings(self): # Some services (route53) model MaxItems as a string type. # We need to be able to handle this case. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( # Note max_items is a string here. paginator.paginate(None, max_items='1').build_full_result(), {'Users': ['User1'], 'NextToken': 'm1'})
def test_max_items_can_be_specified_truncates_response(self): # We're saying we only want 4 items, but notice that the second # page of results returns users 4-6 so we have to truncated # part of that second page. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1", "User2", "User3"], "Marker": "m1"}), (None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}), (None, {"Users": ["User7"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=4).build_full_result(), {'Users': ['User1', 'User2', 'User3', 'User4'], 'NextToken': 'm1___1'})
def test_next_token_with_or_expression(self): self.operation.pagination = { 'output_token': 'NextToken || NextToken2', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.operation, self.operation.pagination) # Verify that despite varying between NextToken and NextToken2 # we still can extract the right next tokens. responses = [ (None, {'NextToken': 'token1'}), (None, {'NextToken2': 'token2'}), # The first match found wins, so because NextToken is # listed before NextToken2 in the 'output_tokens' config, # 'token3' is chosen over 'token4'. (None, {'NextToken': 'token3', 'NextToken2': 'token4'}), (None, {'not_next_token': 'foo'}), ] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'), mock.call(None, NextToken='token3'),])
def test_include_with_nested_result_keys(self): self.paginate_config['result_key'] = 'Result.Key' self.paginate_config['non_aggregate_keys'] = [ 'Outer', 'Result.Inner', ] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([ # The non result keys shows hypothetical # example. This doesn't actually happen, # but in the case where the non result keys # are different across pages, we use the values # from the first page. {'Result': {'Key': ['foo'], 'Inner': 'v1'}, 'Outer': 'v2', 'NextToken': 't1'}, {'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'}, 'Outer': 'v4', 'NextToken': 't2'}, {'Result': {'Key': ['qux'], 'Inner': 'v5'}, 'Outer': 'v6', 'NextToken': 't3'}, ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, {'Outer': 'v2', 'Result': {'Inner': 'v1'}}) expected = { 'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'}, 'Outer': 'v2', } self.assertEqual(actual, expected)
def test_include_with_multiple_result_keys(self): self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2'] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([{ 'ResultKey1': ['a', 'b'], 'ResultKey2': ['u', 'v'], 'NotResultKey': 'a', 'NextToken': 'token1' }, { 'ResultKey1': ['c', 'd'], 'ResultKey2': ['w', 'x'], 'NotResultKey': 'a', 'NextToken': 'token2' }, { 'ResultKey1': ['e', 'f'], 'ResultKey2': ['y', 'z'], 'NotResultKey': 'a', }]) pages = self.paginator.paginate(None) actual = pages.build_full_result() expected = { 'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'], 'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'], 'NotResultKey': 'a', } self.assertEqual(actual, expected)
class TestMultipleTokens(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": ["ListBucketResults.NextKeyMarker", "ListBucketResults.NextUploadIdMarker"], "input_token": ["key_marker", "upload_id_marker"], "result_key": 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_s3_list_multipart_uploads(self): responses = [ (None, {"Foo": [1], "ListBucketResults": {"NextKeyMarker": "key1", "NextUploadIdMarker": "up1"}}), (None, {"Foo": [2], "ListBucketResults": {"NextKeyMarker": "key2", "NextUploadIdMarker": "up2"}}), (None, {"Foo": [3], "ListBucketResults": {"NextKeyMarker": "key3", "NextUploadIdMarker": "up3"}}), (None, {}), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, key_marker='key1', upload_id_marker='up1'), mock.call(None, key_marker='key2', upload_id_marker='up2'), mock.call(None, key_marker='key3', upload_id_marker='up3'), ])
def test_next_token_with_or_expression(self): self.operation.pagination = { 'output_token': 'NextToken || NextToken2', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.operation, self.operation.pagination) # Verify that despite varying between NextToken and NextToken2 # we still can extract the right next tokens. responses = [ (None, { 'NextToken': 'token1' }), (None, { 'NextToken2': 'token2' }), # The first match found wins, so because NextToken is # listed before NextToken2 in the 'output_tokens' config, # 'token3' is chosen over 'token4'. (None, { 'NextToken': 'token3', 'NextToken2': 'token4' }), (None, { 'not_next_token': 'foo' }), ] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'), mock.call(None, NextToken='token3'), ])
def setUp(self): self.operation = mock.Mock() # This is something like what we'd see in RDS. self.paginate_config = { "input_token": "Marker", "output_token": "Marker", "limit_key": "MaxRecords", "result_key": "EngineDefaults.Parameters" } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.responses = [ (None, { "EngineDefaults": { "Parameters": ["One", "Two"] }, "Marker": "m1" }), (None, { "EngineDefaults": { "Parameters": ["Three", "Four"] }, "Marker": "m2" }), (None, { "EngineDefaults": { "Parameters": ["Five"] } }), ]
def test_more_tokens_is_path_expression(self): self.paginate_config = { 'more_results': 'Foo.IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Bar', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { 'Foo': { 'IsTruncated': True }, 'NextToken': 'token1' }), (None, { 'Foo': { 'IsTruncated': False }, 'NextToken': 'token2' }), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, NextToken='token1'), ])
def setUp(self): self.operation = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
def test_resume_next_marker_mid_page(self): # This is a simulation of picking up from the response # from test_max_items_can_be_specified_truncates_response # We got the first 4 users, when we pick up we should get # User5 - User7. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}), (None, {"Users": ["User7"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, starting_token='m1___1').build_full_result(), {'Users': ['User5', 'User6', 'User7']}) self.assertEqual( self.operation.call.call_args_list, [mock.call(None, Marker='m1'), mock.call(None, Marker='m2'),])
def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": ["Users", "Groups"], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
def setUp(self): self.operation = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'ResultKey', 'non_aggregate_keys': ['NotResultKey'], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
def test_include_with_nested_result_keys(self): self.paginate_config['result_key'] = 'Result.Key' self.paginate_config['non_aggregate_keys'] = [ 'Outer', 'Result.Inner', ] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([ # The non result keys shows hypothetical # example. This doesn't actually happen, # but in the case where the non result keys # are different across pages, we use the values # from the first page. { 'Result': { 'Key': ['foo'], 'Inner': 'v1' }, 'Outer': 'v2', 'NextToken': 't1' }, { 'Result': { 'Key': ['bar', 'baz'], 'Inner': 'v3' }, 'Outer': 'v4', 'NextToken': 't2' }, { 'Result': { 'Key': ['qux'], 'Inner': 'v5' }, 'Outer': 'v6', 'NextToken': 't3' }, ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, { 'Outer': 'v2', 'Result': { 'Inner': 'v1' } }) expected = { 'Result': { 'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1' }, 'Outer': 'v2', } self.assertEqual(actual, expected)
def setUp(self): self.operation = mock.Mock() # Probably the most complicated example we'll see: # multiple input/output/result keys. self.paginate_config = { "output_token": ["Marker1", "Marker2"], "input_token": ["InMarker1", "InMarker2"], "result_key": ["Users", "Groups"], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": ["ListBucketResults.NextKeyMarker", "ListBucketResults.NextUploadIdMarker"], "input_token": ["key_marker", "upload_id_marker"], "result_key": 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { 'output_token': [ 'NextMarker || ListBucketResult.Contents[-1].Key'], 'input_token': 'next_marker', 'result_key': 'Contents', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { 'output_token': ['NextMarker || ListBucketResult.Contents[-1].Key'], 'input_token': 'next_marker', 'result_key': 'Contents', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
def setUp(self): self.operation = mock.Mock() self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": ["Users", "Groups"], 'limit_key': 'MaxKeys', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.endpoint = mock.Mock()
def test_max_items_exceeds_actual_amount(self): # Because MaxItems=10 > number of users (3), we should just return # all of the users. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=10).build_full_result(), {'Users': ['User1', 'User2', 'User3']})
def test_max_items_can_be_specified(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=1).build_full_result(), { 'Users': ['User1'], 'NextToken': 'm1' })
def test_next_token_on_page_boundary(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=2).build_full_result(), { 'Users': ['User1', 'User2'], 'NextToken': 'm2' })
class TestPaginatorWithPathExpressions(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { 'output_token': [ 'NextMarker || ListBucketResult.Contents[-1].Key'], 'input_token': 'next_marker', 'result_key': 'Contents', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_s3_list_objects(self): responses = [ (None, {'NextMarker': 'token1'}), (None, {'NextMarker': 'token2'}), (None, {'not_next_token': 'foo'})] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, next_marker='token1'), mock.call(None, next_marker='token2'),]) def test_s3_list_object_complex(self): responses = [ (None, {'NextMarker': 'token1'}), (None, {'ListBucketResult': { 'Contents': [{"Key": "first"}, {"Key": "Last"}]}}), (None, {'not_next_token': 'foo'})] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, next_marker='token1'), mock.call(None, next_marker='Last'),])
class TestExpressionKeyIterators(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something like what we'd see in RDS. self.paginate_config = { "input_token": "Marker", "output_token": "Marker", "limit_key": "MaxRecords", "result_key": "EngineDefaults.Parameters" } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.responses = [ (None, { "EngineDefaults": {"Parameters": ["One", "Two"] }, "Marker": "m1"}), (None, { "EngineDefaults": {"Parameters": ["Three", "Four"] }, "Marker": "m2"}), (None, {"EngineDefaults": {"Parameters": ["Five"]}}), ] def test_result_key_iters(self): self.operation.call.side_effect = self.responses pages = self.paginator.paginate(None) iterators = pages.result_key_iters() self.assertEqual(len(iterators), 1) self.assertEqual(list(iterators[0]), ['One', 'Two', 'Three', 'Four', 'Five']) def test_build_full_result_with_single_key(self): self.operation.call.side_effect = self.responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, { 'EngineDefaults': { 'Parameters': ['One', 'Two', 'Three', 'Four', 'Five'] }, })
def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": [ "ListBucketResults.NextKeyMarker", "ListBucketResults.NextUploadIdMarker" ], "input_token": ["key_marker", "upload_id_marker"], "result_key": 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config)
class TestMultipleTokens(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": [ "ListBucketResults.NextKeyMarker", "ListBucketResults.NextUploadIdMarker" ], "input_token": ["key_marker", "upload_id_marker"], "result_key": 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_s3_list_multipart_uploads(self): responses = [ (None, { "Foo": [1], "ListBucketResults": { "NextKeyMarker": "key1", "NextUploadIdMarker": "up1" } }), (None, { "Foo": [2], "ListBucketResults": { "NextKeyMarker": "key2", "NextUploadIdMarker": "up2" } }), (None, { "Foo": [3], "ListBucketResults": { "NextKeyMarker": "key3", "NextUploadIdMarker": "up3" } }), (None, {}), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, key_marker='key1', upload_id_marker='up1'), mock.call(None, key_marker='key2', upload_id_marker='up2'), mock.call(None, key_marker='key3', upload_id_marker='up3'), ])
def test_resume_next_marker_mid_page(self): # This is a simulation of picking up from the response # from test_max_items_can_be_specified_truncates_response # We got the first 4 users, when we pick up we should get # User5 - User7. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User4", "User5", "User6"], "Marker": "m2" }), (None, { "Users": ["User7"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, starting_token='m1___1').build_full_result(), {'Users': ['User5', 'User6', 'User7']}) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None, Marker='m1'), mock.call(None, Marker='m2'), ])
def test_max_items_can_be_specified_truncates_response(self): # We're saying we only want 4 items, but notice that the second # page of results returns users 4-6 so we have to truncated # part of that second page. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1", "User2", "User3"], "Marker": "m1" }), (None, { "Users": ["User4", "User5", "User6"], "Marker": "m2" }), (None, { "Users": ["User7"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=4).build_full_result(), { 'Users': ['User1', 'User2', 'User3', 'User4'], 'NextToken': 'm1___1' })
def test_more_tokens_is_path_expression(self): self.paginate_config = { 'more_results': 'Foo.IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Bar', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {'Foo': {'IsTruncated': True}, 'NextToken': 'token1'}), (None, {'Foo': {'IsTruncated': False}, 'NextToken': 'token2'}), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, NextToken='token1'),])
def setUp(self): self.operation = mock.Mock() # This is something like what we'd see in RDS. self.paginate_config = { "input_token": "Marker", "output_token": "Marker", "limit_key": "MaxRecords", "result_key": "EngineDefaults.Parameters" } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.responses = [ (None, { "EngineDefaults": {"Parameters": ["One", "Two"] }, "Marker": "m1"}), (None, { "EngineDefaults": {"Parameters": ["Three", "Four"] }, "Marker": "m2"}), (None, {"EngineDefaults": {"Parameters": ["Five"]}}), ]
def test_include_with_multiple_result_keys(self): self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2'] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([ {'ResultKey1': ['a', 'b'], 'ResultKey2': ['u', 'v'], 'NotResultKey': 'a', 'NextToken': 'token1'}, {'ResultKey1': ['c', 'd'], 'ResultKey2': ['w', 'x'], 'NotResultKey': 'a', 'NextToken': 'token2'}, {'ResultKey1': ['e', 'f'], 'ResultKey2': ['y', 'z'], 'NotResultKey': 'a',} ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() expected = { 'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'], 'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'], 'NotResultKey': 'a', } self.assertEqual(actual, expected)
def test_more_tokens(self): # Some pagination configs have a 'more_token' key that # indicate whether or not the results are being paginated. self.paginate_config = { 'more_results': 'IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { 'Foo': [1], 'IsTruncated': True, 'NextToken': 'token1' }), (None, { 'Foo': [2], 'IsTruncated': True, 'NextToken': 'token2' }), (None, { 'Foo': [3], 'IsTruncated': False, 'NextToken': 'token3' }), (None, { 'Foo': [4], 'not_next_token': 'foo' }), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'), ])
def test_more_tokens(self): # Some pagination configs have a 'more_token' key that # indicate whether or not the results are being paginated. self.paginate_config = { 'more_results': 'IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {'Foo': [1], 'IsTruncated': True, 'NextToken': 'token1'}), (None, {'Foo': [2], 'IsTruncated': True, 'NextToken': 'token2'}), (None, {'Foo': [3], 'IsTruncated': False, 'NextToken': 'token3'}), (None, {'Foo': [4], 'not_next_token': 'foo'}), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'),])
class TestPagination(unittest.TestCase): def setUp(self): self.operation = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_result_key_available(self): self.assertEqual( [rk.expression for rk in self.paginator.result_keys], ['Foo'] ) def test_no_next_token(self): response = {'not_the_next_token': 'foobar'} self.operation.call.return_value = None, response actual = list(self.paginator.paginate(None)) self.assertEqual(actual, [(None, {'not_the_next_token': 'foobar'})]) def test_next_token_in_response(self): responses = [(None, {'NextToken': 'token1'}), (None, {'NextToken': 'token2'}), (None, {'not_next_token': 'foo'})] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None)) self.assertEqual(actual, responses) # The first call has no next token, the second and third call should # have 'token1' and 'token2' respectively. self.assertEqual(self.operation.call.call_args_list, [mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2')]) def test_any_passed_in_args_are_unmodified(self): responses = [(None, {'NextToken': 'token1'}), (None, {'NextToken': 'token2'}), (None, {'not_next_token': 'foo'})] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None, Foo='foo', Bar='bar')) self.assertEqual(actual, responses) self.assertEqual( self.operation.call.call_args_list, [mock.call(None, Foo='foo', Bar='bar'), mock.call(None, Foo='foo', Bar='bar', NextToken='token1'), mock.call(None, Foo='foo', Bar='bar', NextToken='token2')]) def test_exception_raised_if_same_next_token(self): responses = [(None, {'NextToken': 'token1'}), (None, {'NextToken': 'token2'}), (None, {'NextToken': 'token2'})] self.operation.call.side_effect = responses with self.assertRaises(PaginationError): list(self.paginator.paginate(None)) def test_next_token_with_or_expression(self): self.operation.pagination = { 'output_token': 'NextToken || NextToken2', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.operation, self.operation.pagination) # Verify that despite varying between NextToken and NextToken2 # we still can extract the right next tokens. responses = [ (None, {'NextToken': 'token1'}), (None, {'NextToken2': 'token2'}), # The first match found wins, so because NextToken is # listed before NextToken2 in the 'output_tokens' config, # 'token3' is chosen over 'token4'. (None, {'NextToken': 'token3', 'NextToken2': 'token4'}), (None, {'not_next_token': 'foo'}), ] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'), mock.call(None, NextToken='token3'),]) def test_more_tokens(self): # Some pagination configs have a 'more_token' key that # indicate whether or not the results are being paginated. self.paginate_config = { 'more_results': 'IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {'Foo': [1], 'IsTruncated': True, 'NextToken': 'token1'}), (None, {'Foo': [2], 'IsTruncated': True, 'NextToken': 'token2'}), (None, {'Foo': [3], 'IsTruncated': False, 'NextToken': 'token3'}), (None, {'Foo': [4], 'not_next_token': 'foo'}), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'),]) def test_more_tokens_is_path_expression(self): self.paginate_config = { 'more_results': 'Foo.IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Bar', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {'Foo': {'IsTruncated': True}, 'NextToken': 'token1'}), (None, {'Foo': {'IsTruncated': False}, 'NextToken': 'token2'}), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual( self.operation.call.call_args_list, [mock.call(None), mock.call(None, NextToken='token1'),])
class TestKeyIterators(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": "Users" } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_result_key_iters(self): responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) iterators = pages.result_key_iters() self.assertEqual(len(iterators), 1) self.assertEqual(list(iterators[0]), ["User1", "User2", "User3"]) def test_build_full_result_with_single_key(self): responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']}) def test_max_items_can_be_specified(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=1).build_full_result(), { 'Users': ['User1'], 'NextToken': 'm1' }) def test_max_items_as_strings(self): # Some services (route53) model MaxItems as a string type. # We need to be able to handle this case. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( # Note max_items is a string here. paginator.paginate(None, max_items='1').build_full_result(), { 'Users': ['User1'], 'NextToken': 'm1' }) def test_next_token_on_page_boundary(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=2).build_full_result(), { 'Users': ['User1', 'User2'], 'NextToken': 'm2' }) def test_max_items_can_be_specified_truncates_response(self): # We're saying we only want 4 items, but notice that the second # page of results returns users 4-6 so we have to truncated # part of that second page. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1", "User2", "User3"], "Marker": "m1" }), (None, { "Users": ["User4", "User5", "User6"], "Marker": "m2" }), (None, { "Users": ["User7"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=4).build_full_result(), { 'Users': ['User1', 'User2', 'User3', 'User4'], 'NextToken': 'm1___1' }) def test_resume_next_marker_mid_page(self): # This is a simulation of picking up from the response # from test_max_items_can_be_specified_truncates_response # We got the first 4 users, when we pick up we should get # User5 - User7. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User4", "User5", "User6"], "Marker": "m2" }), (None, { "Users": ["User7"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, starting_token='m1___1').build_full_result(), {'Users': ['User5', 'User6', 'User7']}) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None, Marker='m1'), mock.call(None, Marker='m2'), ]) def test_max_items_exceeds_actual_amount(self): # Because MaxItems=10 > number of users (3), we should just return # all of the users. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=10).build_full_result(), {'Users': ['User1', 'User2', 'User3']}) def test_bad_input_tokens(self): responses = [ (None, { "Users": ["User1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Marker": "m2" }), (None, { "Users": ["User3"] }), ] self.operation.call.side_effect = responses with self.assertRaisesRegexp(ValueError, 'Bad starting token'): self.paginator.paginate( None, starting_token='bad___notanint').build_full_result()
class TestMultipleResultKeys(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": ["Users", "Groups"], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_build_full_result_with_multiple_result_keys(self): responses = [ (None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"}), (None, {"Users": ["User3"], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, {"Users": ['User1', 'User2', 'User3'], "Groups": ['Group1', 'Group2', 'Group3']}) def test_build_full_result_with_different_length_result_keys(self): responses = [ (None, {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"}), # Then we stop getting "Users" output, but we get more "Groups" (None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}), (None, {"Users": [], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, {"Users": ['User1'], "Groups": ['Group1', 'Group2', 'Group3']}) def test_build_full_result_with_zero_length_result_key(self): responses = [ # In this case the 'Users' key is always empty but we should # have a 'Users' key in the output, it should just have an # empty list for a value. (None, {"Users": [], "Groups": ["Group1"], "Marker": "m1"}), (None, {"Users": [], "Groups": ["Group2"], "Marker": "m2"}), (None, {"Users": [], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, {"Users": [], "Groups": ['Group1', 'Group2', 'Group3']}) def test_build_result_with_secondary_keys(self): responses = [ (None, {"Users": ["User1", "User2"], "Groups": ["Group1", "Group2"], "Marker": "m1"}), (None, {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"}), (None, {"Users": ["User4"], "Groups": ["Group4"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=1) complete = pages.build_full_result() self.assertEqual(complete, {"Users": ["User1"], "Groups": ["Group1", "Group2"], "NextToken": "None___1"}) def test_resume_with_secondary_keys(self): # This is simulating a continutation of the previous test, # test_build_result_with_secondary_keys. We use the # token specified in the response "None___1" to continue where we # left off. responses = [ (None, {"Users": ["User1", "User2"], "Groups": ["Group1", "Group2"], "Marker": "m1"}), (None, {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"}), (None, {"Users": ["User4"], "Groups": ["Group4"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=1, starting_token="None___1") complete = pages.build_full_result() # Note that the secondary keys ("Groups") are all truncated because # they were in the original (first) response. self.assertEqual(complete, {"Users": ["User2"], "Groups": [], "NextToken": "m1"})
class TestIncludeNonResultKeys(unittest.TestCase): maxDiff = None def setUp(self): self.operation = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'ResultKey', 'non_aggregate_keys': ['NotResultKey'], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def set_responses(self, responses): complete_responses = [] for response in responses: complete_responses.append((None, response)) self.operation.call.side_effect = complete_responses def test_include_non_aggregate_keys(self): self.set_responses([ {'ResultKey': ['foo'], 'NotResultKey': 'a', 'NextToken': 't1'}, {'ResultKey': ['bar'], 'NotResultKey': 'a', 'NextToken': 't2'}, {'ResultKey': ['baz'], 'NotResultKey': 'a'}, ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, {'NotResultKey': 'a'}) expected = { 'ResultKey': ['foo', 'bar', 'baz'], 'NotResultKey': 'a', } self.assertEqual(actual, expected) def test_include_with_multiple_result_keys(self): self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2'] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([ {'ResultKey1': ['a', 'b'], 'ResultKey2': ['u', 'v'], 'NotResultKey': 'a', 'NextToken': 'token1'}, {'ResultKey1': ['c', 'd'], 'ResultKey2': ['w', 'x'], 'NotResultKey': 'a', 'NextToken': 'token2'}, {'ResultKey1': ['e', 'f'], 'ResultKey2': ['y', 'z'], 'NotResultKey': 'a',} ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() expected = { 'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'], 'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'], 'NotResultKey': 'a', } self.assertEqual(actual, expected) def test_include_with_nested_result_keys(self): self.paginate_config['result_key'] = 'Result.Key' self.paginate_config['non_aggregate_keys'] = [ 'Outer', 'Result.Inner', ] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([ # The non result keys shows hypothetical # example. This doesn't actually happen, # but in the case where the non result keys # are different across pages, we use the values # from the first page. {'Result': {'Key': ['foo'], 'Inner': 'v1'}, 'Outer': 'v2', 'NextToken': 't1'}, {'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'}, 'Outer': 'v4', 'NextToken': 't2'}, {'Result': {'Key': ['qux'], 'Inner': 'v5'}, 'Outer': 'v6', 'NextToken': 't3'}, ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, {'Outer': 'v2', 'Result': {'Inner': 'v1'}}) expected = { 'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'}, 'Outer': 'v2', } self.assertEqual(actual, expected)
class TestIncludeNonResultKeys(unittest.TestCase): maxDiff = None def setUp(self): self.operation = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'ResultKey', 'non_aggregate_keys': ['NotResultKey'], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def set_responses(self, responses): complete_responses = [] for response in responses: complete_responses.append((None, response)) self.operation.call.side_effect = complete_responses def test_include_non_aggregate_keys(self): self.set_responses([ { 'ResultKey': ['foo'], 'NotResultKey': 'a', 'NextToken': 't1' }, { 'ResultKey': ['bar'], 'NotResultKey': 'a', 'NextToken': 't2' }, { 'ResultKey': ['baz'], 'NotResultKey': 'a' }, ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, {'NotResultKey': 'a'}) expected = { 'ResultKey': ['foo', 'bar', 'baz'], 'NotResultKey': 'a', } self.assertEqual(actual, expected) def test_include_with_multiple_result_keys(self): self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2'] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([{ 'ResultKey1': ['a', 'b'], 'ResultKey2': ['u', 'v'], 'NotResultKey': 'a', 'NextToken': 'token1' }, { 'ResultKey1': ['c', 'd'], 'ResultKey2': ['w', 'x'], 'NotResultKey': 'a', 'NextToken': 'token2' }, { 'ResultKey1': ['e', 'f'], 'ResultKey2': ['y', 'z'], 'NotResultKey': 'a', }]) pages = self.paginator.paginate(None) actual = pages.build_full_result() expected = { 'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'], 'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'], 'NotResultKey': 'a', } self.assertEqual(actual, expected) def test_include_with_nested_result_keys(self): self.paginate_config['result_key'] = 'Result.Key' self.paginate_config['non_aggregate_keys'] = [ 'Outer', 'Result.Inner', ] self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) self.set_responses([ # The non result keys shows hypothetical # example. This doesn't actually happen, # but in the case where the non result keys # are different across pages, we use the values # from the first page. { 'Result': { 'Key': ['foo'], 'Inner': 'v1' }, 'Outer': 'v2', 'NextToken': 't1' }, { 'Result': { 'Key': ['bar', 'baz'], 'Inner': 'v3' }, 'Outer': 'v4', 'NextToken': 't2' }, { 'Result': { 'Key': ['qux'], 'Inner': 'v5' }, 'Outer': 'v6', 'NextToken': 't3' }, ]) pages = self.paginator.paginate(None) actual = pages.build_full_result() self.assertEqual(pages.non_aggregate_part, { 'Outer': 'v2', 'Result': { 'Inner': 'v1' } }) expected = { 'Result': { 'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1' }, 'Outer': 'v2', } self.assertEqual(actual, expected)
class TestMultipleResultKeys(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": ["Users", "Groups"], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_build_full_result_with_multiple_result_keys(self): responses = [ (None, { "Users": ["User1"], "Groups": ["Group1"], "Marker": "m1" }), (None, { "Users": ["User2"], "Groups": ["Group2"], "Marker": "m2" }), (None, { "Users": ["User3"], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual( complete, { "Users": ['User1', 'User2', 'User3'], "Groups": ['Group1', 'Group2', 'Group3'] }) def test_build_full_result_with_different_length_result_keys(self): responses = [ (None, { "Users": ["User1"], "Groups": ["Group1"], "Marker": "m1" }), # Then we stop getting "Users" output, but we get more "Groups" (None, { "Users": [], "Groups": ["Group2"], "Marker": "m2" }), (None, { "Users": [], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, { "Users": ['User1'], "Groups": ['Group1', 'Group2', 'Group3'] }) def test_build_full_result_with_zero_length_result_key(self): responses = [ # In this case the 'Users' key is always empty but we should # have a 'Users' key in the output, it should just have an # empty list for a value. (None, { "Users": [], "Groups": ["Group1"], "Marker": "m1" }), (None, { "Users": [], "Groups": ["Group2"], "Marker": "m2" }), (None, { "Users": [], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, { "Users": [], "Groups": ['Group1', 'Group2', 'Group3'] }) def test_build_result_with_secondary_keys(self): responses = [ (None, { "Users": ["User1", "User2"], "Groups": ["Group1", "Group2"], "Marker": "m1" }), (None, { "Users": ["User3"], "Groups": ["Group3"], "Marker": "m2" }), (None, { "Users": ["User4"], "Groups": ["Group4"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=1) complete = pages.build_full_result() self.assertEqual( complete, { "Users": ["User1"], "Groups": ["Group1", "Group2"], "NextToken": "None___1" }) def test_resume_with_secondary_keys(self): # This is simulating a continutation of the previous test, # test_build_result_with_secondary_keys. We use the # token specified in the response "None___1" to continue where we # left off. responses = [ (None, { "Users": ["User1", "User2"], "Groups": ["Group1", "Group2"], "Marker": "m1" }), (None, { "Users": ["User3"], "Groups": ["Group3"], "Marker": "m2" }), (None, { "Users": ["User4"], "Groups": ["Group4"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=1, starting_token="None___1") complete = pages.build_full_result() # Note that the secondary keys ("Groups") are all truncated because # they were in the original (first) response. self.assertEqual(complete, { "Users": ["User2"], "Groups": [], "NextToken": "m1" })
class TestPagination(unittest.TestCase): def setUp(self): self.operation = mock.Mock() self.paginate_config = { 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_result_key_available(self): self.assertEqual([rk.expression for rk in self.paginator.result_keys], ['Foo']) def test_no_next_token(self): response = {'not_the_next_token': 'foobar'} self.operation.call.return_value = None, response actual = list(self.paginator.paginate(None)) self.assertEqual(actual, [(None, {'not_the_next_token': 'foobar'})]) def test_next_token_in_response(self): responses = [(None, { 'NextToken': 'token1' }), (None, { 'NextToken': 'token2' }), (None, { 'not_next_token': 'foo' })] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None)) self.assertEqual(actual, responses) # The first call has no next token, the second and third call should # have 'token1' and 'token2' respectively. self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2') ]) def test_any_passed_in_args_are_unmodified(self): responses = [(None, { 'NextToken': 'token1' }), (None, { 'NextToken': 'token2' }), (None, { 'not_next_token': 'foo' })] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None, Foo='foo', Bar='bar')) self.assertEqual(actual, responses) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None, Foo='foo', Bar='bar'), mock.call(None, Foo='foo', Bar='bar', NextToken='token1'), mock.call(None, Foo='foo', Bar='bar', NextToken='token2') ]) def test_exception_raised_if_same_next_token(self): responses = [(None, { 'NextToken': 'token1' }), (None, { 'NextToken': 'token2' }), (None, { 'NextToken': 'token2' })] self.operation.call.side_effect = responses with self.assertRaises(PaginationError): list(self.paginator.paginate(None)) def test_next_token_with_or_expression(self): self.operation.pagination = { 'output_token': 'NextToken || NextToken2', 'input_token': 'NextToken', 'result_key': 'Foo', } self.paginator = Paginator(self.operation, self.operation.pagination) # Verify that despite varying between NextToken and NextToken2 # we still can extract the right next tokens. responses = [ (None, { 'NextToken': 'token1' }), (None, { 'NextToken2': 'token2' }), # The first match found wins, so because NextToken is # listed before NextToken2 in the 'output_tokens' config, # 'token3' is chosen over 'token4'. (None, { 'NextToken': 'token3', 'NextToken2': 'token4' }), (None, { 'not_next_token': 'foo' }), ] self.operation.call.side_effect = responses actual = list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'), mock.call(None, NextToken='token3'), ]) def test_more_tokens(self): # Some pagination configs have a 'more_token' key that # indicate whether or not the results are being paginated. self.paginate_config = { 'more_results': 'IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Foo', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { 'Foo': [1], 'IsTruncated': True, 'NextToken': 'token1' }), (None, { 'Foo': [2], 'IsTruncated': True, 'NextToken': 'token2' }), (None, { 'Foo': [3], 'IsTruncated': False, 'NextToken': 'token3' }), (None, { 'Foo': [4], 'not_next_token': 'foo' }), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, NextToken='token1'), mock.call(None, NextToken='token2'), ]) def test_more_tokens_is_path_expression(self): self.paginate_config = { 'more_results': 'Foo.IsTruncated', 'output_token': 'NextToken', 'input_token': 'NextToken', 'result_key': 'Bar', } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, { 'Foo': { 'IsTruncated': True }, 'NextToken': 'token1' }), (None, { 'Foo': { 'IsTruncated': False }, 'NextToken': 'token2' }), ] self.operation.call.side_effect = responses list(self.paginator.paginate(None)) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None), mock.call(None, NextToken='token1'), ])
class TestMultipleInputKeys(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # Probably the most complicated example we'll see: # multiple input/output/result keys. self.paginate_config = { "output_token": ["Marker1", "Marker2"], "input_token": ["InMarker1", "InMarker2"], "result_key": ["Users", "Groups"], } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_build_full_result_with_multiple_input_keys(self): responses = [ (None, { "Users": ["User1", "User2"], "Groups": ["Group1"], "Marker1": "m1", "Marker2": "m2" }), (None, { "Users": ["User3", "User4"], "Groups": ["Group2"], "Marker1": "m3", "Marker2": "m4" }), (None, { "Users": ["User5"], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=3) complete = pages.build_full_result() self.assertEqual( complete, { "Users": ['User1', 'User2', 'User3'], "Groups": ['Group1', 'Group2'], "NextToken": "m1___m2___1" }) def test_resume_with_multiple_input_keys(self): responses = [ (None, { "Users": ["User3", "User4"], "Groups": ["Group2"], "Marker1": "m3", "Marker2": "m4" }), (None, { "Users": ["User5"], "Groups": ["Group3"], }), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None, max_items=1, starting_token='m1___m2___1') complete = pages.build_full_result() self.assertEqual(complete, { "Users": ['User4'], "Groups": [], "NextToken": "m3___m4" }) self.assertEqual(self.operation.call.call_args_list, [ mock.call(None, InMarker1='m1', InMarker2='m2'), ]) def test_result_key_exposed_on_paginator(self): self.assertEqual([rk.expression for rk in self.paginator.result_keys], ['Users', 'Groups']) def test_result_key_exposed_on_page_iterator(self): pages = self.paginator.paginate(None, max_items=3) self.assertEqual([rk.expression for rk in pages.result_keys], ['Users', 'Groups'])
class TestKeyIterators(unittest.TestCase): def setUp(self): self.operation = mock.Mock() # This is something we'd see in s3 pagination. self.paginate_config = { "output_token": "Marker", "input_token": "Marker", "result_key": "Users" } self.operation.pagination = self.paginate_config self.paginator = Paginator(self.operation, self.paginate_config) def test_result_key_iters(self): responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) iterators = pages.result_key_iters() self.assertEqual(len(iterators), 1) self.assertEqual(list(iterators[0]), ["User1", "User2", "User3"]) def test_build_full_result_with_single_key(self): responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses pages = self.paginator.paginate(None) complete = pages.build_full_result() self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']}) def test_max_items_can_be_specified(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=1).build_full_result(), {'Users': ['User1'], 'NextToken': 'm1'}) def test_max_items_as_strings(self): # Some services (route53) model MaxItems as a string type. # We need to be able to handle this case. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( # Note max_items is a string here. paginator.paginate(None, max_items='1').build_full_result(), {'Users': ['User1'], 'NextToken': 'm1'}) def test_next_token_on_page_boundary(self): paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=2).build_full_result(), {'Users': ['User1', 'User2'], 'NextToken': 'm2'}) def test_max_items_can_be_specified_truncates_response(self): # We're saying we only want 4 items, but notice that the second # page of results returns users 4-6 so we have to truncated # part of that second page. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1", "User2", "User3"], "Marker": "m1"}), (None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}), (None, {"Users": ["User7"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=4).build_full_result(), {'Users': ['User1', 'User2', 'User3', 'User4'], 'NextToken': 'm1___1'}) def test_resume_next_marker_mid_page(self): # This is a simulation of picking up from the response # from test_max_items_can_be_specified_truncates_response # We got the first 4 users, when we pick up we should get # User5 - User7. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User4", "User5", "User6"], "Marker": "m2"}), (None, {"Users": ["User7"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, starting_token='m1___1').build_full_result(), {'Users': ['User5', 'User6', 'User7']}) self.assertEqual( self.operation.call.call_args_list, [mock.call(None, Marker='m1'), mock.call(None, Marker='m2'),]) def test_max_items_exceeds_actual_amount(self): # Because MaxItems=10 > number of users (3), we should just return # all of the users. paginator = Paginator(self.operation, self.paginate_config) responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses self.assertEqual( paginator.paginate(None, max_items=10).build_full_result(), {'Users': ['User1', 'User2', 'User3']}) def test_bad_input_tokens(self): responses = [ (None, {"Users": ["User1"], "Marker": "m1"}), (None, {"Users": ["User2"], "Marker": "m2"}), (None, {"Users": ["User3"]}), ] self.operation.call.side_effect = responses with self.assertRaisesRegexp(ValueError, 'Bad starting token'): self.paginator.paginate( None, starting_token='bad___notanint').build_full_result()