class TestSearchOverResults(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        self.paginate_config = {
            'more_results': 'IsTruncated',
            'output_token': 'NextToken',
            'input_token': 'NextToken',
            'result_key': 'Foo',
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {'Foo': [{'a': 1}, {'b': 2}],
             'IsTruncated': True, 'NextToken': '1'},
            {'Foo': [{'a': 3}, {'b': 4}],
             'IsTruncated': True, 'NextToken': '2'},
            {'Foo': [{'a': 5}], 'IsTruncated': False, 'NextToken': '3'}
        ]
        self.method.side_effect = responses

    def test_yields_non_list_values(self):
        result = list(self.paginator.paginate().search('Foo[0].a'))
        self.assertEqual([1, 3, 5], result)

    def test_yields_individual_list_values(self):
        result = list(self.paginator.paginate().search('Foo[].*[]'))
        self.assertEqual([1, 2, 3, 4, 5], result)

    def test_empty_when_no_match(self):
        result = list(self.paginator.paginate().search('Foo[].qux'))
        self.assertEqual([], result)

    def test_no_yield_when_no_match_on_page(self):
        result = list(self.paginator.paginate().search('Foo[].b'))
        self.assertEqual([2, 4], result)
class TestIncludeResultKeys(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        self.paginate_config = {
            'output_token': 'Marker',
            'input_token': 'Marker',
            'result_key': ['ResultKey', 'Count', 'Log'],
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_different_kinds_of_result_key(self):
        self.method.side_effect = [
            {'ResultKey': ['a'], 'Count': 1, 'Log': 'x', 'Marker': 'a'},
            {'not_a_result_key': 'this page will be ignored', 'Marker': '_'},
            {'ResultKey': ['b', 'c'], 'Count': 2, 'Log': 'y', 'Marker': 'b'},
            {'ResultKey': ['d', 'e', 'f'], 'Count': 3, 'Log': 'z'},
        ]
        pages = self.paginator.paginate()
        expected = {
            'ResultKey': ['a', 'b', 'c', 'd', 'e', 'f'],
            'Count': 6,
            'Log': 'xyz',
        }
        self.assertEqual(pages.build_full_result(), expected)

    def test_result_key_is_missing(self):
        self.method.side_effect = [
            {'not_a_result_key': 'this page will be ignored', 'Marker': '_'},
            {'neither_this_one': 'this page will be ignored, too'},
        ]
        pages = self.paginator.paginate()
        expected = {}
        self.assertEqual(pages.build_full_result(), expected)
class TestMultipleTokens(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        # This is something we'd see in s3 pagination.
        self.paginate_config = {
            "output_token": ["ListBucketResults.NextKeyMarker",
                             "ListBucketResults.NextUploadIdMarker"],
            "input_token": ["key_marker", "upload_id_marker"],
            "result_key": 'Foo',
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_s3_list_multipart_uploads(self):
        responses = [
            {"Foo": [1], "ListBucketResults": {"NextKeyMarker": "key1",
                                               "NextUploadIdMarker": "up1"}},
            {"Foo": [2], "ListBucketResults": {"NextKeyMarker": "key2",
                                               "NextUploadIdMarker": "up2"}},
            {"Foo": [3], "ListBucketResults": {"NextKeyMarker": "key3",
                                               "NextUploadIdMarker": "up3"}},
            {}
        ]
        self.method.side_effect = responses
        list(self.paginator.paginate())
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(),
             mock.call(key_marker='key1', upload_id_marker='up1'),
             mock.call(key_marker='key2', upload_id_marker='up2'),
             mock.call(key_marker='key3', upload_id_marker='up3'),
             ])
 def test_next_token_on_page_boundary(self):
     paginator = Paginator(self.method, self.paginate_config)
     responses = [
         {"Users": ["User1"], "Marker": "m1"},
         {"Users": ["User2"], "Marker": "m2"},
         {"Users": ["User3"]},
     ]
     self.method.side_effect = responses
     expected_token = encode_token({"Marker": "m2"})
     self.assertEqual(
         paginator.paginate(
             PaginationConfig={'MaxItems': 2}).build_full_result(),
         {'Users': ['User1', 'User2'], 'NextToken': expected_token})
 def test_max_items_can_be_specified(self):
     paginator = Paginator(self.method, self.paginate_config)
     responses = [
         {"Users": ["User1"], "Marker": "m1"},
         {"Users": ["User2"], "Marker": "m2"},
         {"Users": ["User3"]},
     ]
     self.method.side_effect = responses
     expected_token = encode_token({"Marker": "m1"})
     self.assertEqual(
         paginator.paginate(
             PaginationConfig={'MaxItems': 1}).build_full_result(),
         {'Users': ['User1'], 'NextToken': expected_token})
class TestPaginatorWithPathExpressions(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        # This is something we'd see in s3 pagination.
        self.paginate_config = {
            'output_token': [
                'NextMarker || ListBucketResult.Contents[-1].Key'],
            'input_token': 'next_marker',
            'result_key': 'Contents',
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_s3_list_objects(self):
        responses = [
            {'NextMarker': 'token1'},
            {'NextMarker': 'token2'},
            {'not_next_token': 'foo'}]
        self.method.side_effect = responses
        list(self.paginator.paginate())
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(),
             mock.call(next_marker='token1'),
             mock.call(next_marker='token2')])

    def test_s3_list_object_complex(self):
        responses = [
            {'NextMarker': 'token1'},
            {'ListBucketResult': {
                'Contents': [{"Key": "first"}, {"Key": "Last"}]}},
            {'not_next_token': 'foo'}]
        self.method.side_effect = responses
        list(self.paginator.paginate())
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(),
             mock.call(next_marker='token1'),
             mock.call(next_marker='Last')])
 def test_max_items_exceeds_actual_amount(self):
     # Because MaxItems=10 > number of users (3), we should just return
     # all of the users.
     paginator = Paginator(self.method, self.paginate_config)
     responses = [
         {"Users": ["User1"], "Marker": "m1"},
         {"Users": ["User2"], "Marker": "m2"},
         {"Users": ["User3"]},
     ]
     self.method.side_effect = responses
     self.assertEqual(
         paginator.paginate(
             PaginationConfig={'MaxItems': 10}).build_full_result(),
         {'Users': ['User1', 'User2', 'User3']})
class TestPaginatorPageSize(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        self.paginate_config = {
            "output_token": "Marker",
            "input_token": "Marker",
            "result_key": ["Users", "Groups"],
            'limit_key': 'MaxKeys',
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        self.endpoint = mock.Mock()

    def test_no_page_size(self):
        kwargs = {'arg1': 'foo', 'arg2': 'bar'}
        ref_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
        pages = self.paginator.paginate(**kwargs)
        pages._inject_starting_params(kwargs)
        self.assertEqual(kwargs, ref_kwargs)

    def test_page_size(self):
        kwargs = {'arg1': 'foo', 'arg2': 'bar',
                  'PaginationConfig': {'PageSize': 5}}
        extracted_kwargs = {'arg1': 'foo', 'arg2': 'bar'}
        # Note that ``MaxKeys`` in ``setUp()`` is the parameter used for
        # the page size for pagination.
        ref_kwargs = {'arg1': 'foo', 'arg2': 'bar', 'MaxKeys': 5}
        pages = self.paginator.paginate(**kwargs)
        pages._inject_starting_params(extracted_kwargs)
        self.assertEqual(extracted_kwargs, ref_kwargs)

    def test_page_size_incorrectly_provided(self):
        kwargs = {'arg1': 'foo', 'arg2': 'bar',
                  'PaginationConfig': {'PageSize': 5}}
        del self.paginate_config['limit_key']

        with self.assertRaises(PaginationError):
            self.paginator.paginate(**kwargs)
class TestExpressionKeyIterators(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        # This is something like what we'd see in RDS.
        self.paginate_config = {
            "input_token": "Marker",
            "output_token": "Marker",
            "limit_key": "MaxRecords",
            "result_key": "EngineDefaults.Parameters"
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        self.responses = [
            {"EngineDefaults": {"Parameters": ["One", "Two"]},
             "Marker": "m1"},
            {"EngineDefaults": {"Parameters": ["Three", "Four"]},
             "Marker": "m2"},
            {"EngineDefaults": {"Parameters": ["Five"]}}
        ]

    def test_result_key_iters(self):
        self.method.side_effect = self.responses
        pages = self.paginator.paginate()
        iterators = pages.result_key_iters()
        self.assertEqual(len(iterators), 1)
        self.assertEqual(list(iterators[0]),
                         ['One', 'Two', 'Three', 'Four', 'Five'])

    def test_build_full_result_with_single_key(self):
        self.method.side_effect = self.responses
        pages = self.paginator.paginate()
        complete = pages.build_full_result()
        self.assertEqual(complete, {
            'EngineDefaults': {
                'Parameters': ['One', 'Two', 'Three', 'Four', 'Five']
            },
        })
 def test_max_items_can_be_specified_truncates_response(self):
     # We're saying we only want 4 items, but notice that the second
     # page of results returns users 4-6 so we have to truncated
     # part of that second page.
     paginator = Paginator(self.method, self.paginate_config)
     responses = [
         {"Users": ["User1", "User2", "User3"], "Marker": "m1"},
         {"Users": ["User4", "User5", "User6"], "Marker": "m2"},
         {"Users": ["User7"]},
     ]
     self.method.side_effect = responses
     expected_token = encode_token(
         {"Marker": "m1", "boto_truncate_amount": 1})
     self.assertEqual(
         paginator.paginate(
             PaginationConfig={'MaxItems': 4}).build_full_result(),
         {'Users': ['User1', 'User2', 'User3', 'User4'],
          'NextToken': expected_token})
class TestOptionalTokens(unittest.TestCase):
    """
    Tests a paginator with an optional output token.

    If this gets left in the request params from a previous page, the API 
    will skip over a record.

    """
    def setUp(self):
        self.method = mock.Mock()
        self.paginate_config = {
            "output_token": ["NextRecordName",
                             "NextRecordType",
                             "NextRecordIdentifier"],
            "input_token": ["StartRecordName",
                            "StartRecordType",
                            "StartRecordIdentifier"],
            "result_key": 'Foo',
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_clean_token(self):
        responses = [
            {"Foo": [1],
             "IsTruncated": True,
             "NextRecordName": "aaa.example.com",
             "NextRecordType": "A",
             "NextRecordIdentifier": "id"},
            {"Foo": [2],
             "IsTruncated": True,
             "NextRecordName": "bbb.example.com",
             "NextRecordType": "A"},
            {"Foo": [3],
             "IsTruncated": False},
        ]
        self.method.side_effect = responses
        list(self.paginator.paginate())
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(),
             mock.call(StartRecordName='aaa.example.com', StartRecordType='A',
                       StartRecordIdentifier='id'),
             mock.call(StartRecordName='bbb.example.com', StartRecordType='A')
             ])
 def test_resume_next_marker_mid_page(self):
     # This is a simulation of picking up from the response
     # from test_MaxItems_can_be_specified_truncates_response
     # We got the first 4 users, when we pick up we should get
     # User5 - User7.
     paginator = Paginator(self.method, self.paginate_config)
     responses = [
         {"Users": ["User4", "User5", "User6"], "Marker": "m2"},
         {"Users": ["User7"]},
     ]
     self.method.side_effect = responses
     starting_token = encode_token(
         {"Marker": "m1", "boto_truncate_amount": 1})
     pagination_config = {'StartingToken': starting_token}
     self.assertEqual(
         paginator.paginate(
             PaginationConfig=pagination_config).build_full_result(),
         {'Users': ['User5', 'User6', 'User7']})
     self.assertEqual(
         self.method.call_args_list,
         [mock.call(Marker='m1'),
          mock.call(Marker='m2')])
class TestIncludeNonResultKeys(unittest.TestCase):
    maxDiff = None

    def setUp(self):
        self.method = mock.Mock()
        self.paginate_config = {
            'output_token': 'NextToken',
            'input_token': 'NextToken',
            'result_key': 'ResultKey',
            'non_aggregate_keys': ['NotResultKey'],
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_include_non_aggregate_keys(self):
        self.method.side_effect = [
            {'ResultKey': ['foo'], 'NotResultKey': 'a', 'NextToken': 't1'},
            {'ResultKey': ['bar'], 'NotResultKey': 'a', 'NextToken': 't2'},
            {'ResultKey': ['baz'], 'NotResultKey': 'a'},
        ]
        pages = self.paginator.paginate()
        actual = pages.build_full_result()
        self.assertEqual(pages.non_aggregate_part, {'NotResultKey': 'a'})
        expected = {
            'ResultKey': ['foo', 'bar', 'baz'],
            'NotResultKey': 'a',
        }
        self.assertEqual(actual, expected)

    def test_include_with_multiple_result_keys(self):
        self.paginate_config['result_key'] = ['ResultKey1', 'ResultKey2']
        self.paginator = Paginator(self.method, self.paginate_config)
        self.method.side_effect = [
            {'ResultKey1': ['a', 'b'], 'ResultKey2': ['u', 'v'],
             'NotResultKey': 'a', 'NextToken': 'token1'},
            {'ResultKey1': ['c', 'd'], 'ResultKey2': ['w', 'x'],
             'NotResultKey': 'a', 'NextToken': 'token2'},
            {'ResultKey1': ['e', 'f'], 'ResultKey2': ['y', 'z'],
             'NotResultKey': 'a'}
        ]
        pages = self.paginator.paginate()
        actual = pages.build_full_result()
        expected = {
            'ResultKey1': ['a', 'b', 'c', 'd', 'e', 'f'],
            'ResultKey2': ['u', 'v', 'w', 'x', 'y', 'z'],
            'NotResultKey': 'a',
        }
        self.assertEqual(actual, expected)

    def test_include_with_nested_result_keys(self):
        self.paginate_config['result_key'] = 'Result.Key'
        self.paginate_config['non_aggregate_keys'] = [
            'Outer', 'Result.Inner',
        ]
        self.paginator = Paginator(self.method, self.paginate_config)
        self.method.side_effect = [
            # The non result keys shows hypothetical
            # example.  This doesn't actually happen,
            # but in the case where the non result keys
            # are different across pages, we use the values
            # from the first page.
            {'Result': {'Key': ['foo'], 'Inner': 'v1'},
             'Outer': 'v2', 'NextToken': 't1'},
            {'Result': {'Key': ['bar', 'baz'], 'Inner': 'v3'},
             'Outer': 'v4', 'NextToken': 't2'},
            {'Result': {'Key': ['qux'], 'Inner': 'v5'},
             'Outer': 'v6', 'NextToken': 't3'},
        ]
        pages = self.paginator.paginate()
        actual = pages.build_full_result()
        self.assertEqual(pages.non_aggregate_part,
                         {'Outer': 'v2', 'Result': {'Inner': 'v1'}})
        expected = {
            'Result': {'Key': ['foo', 'bar', 'baz', 'qux'], 'Inner': 'v1'},
            'Outer': 'v2',
        }
        self.assertEqual(actual, expected)
class TestMultipleInputKeys(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        # Probably the most complicated example we'll see:
        # multiple input/output/result keys.
        self.paginate_config = {
            "output_token": ["Marker1", "Marker2"],
            "input_token": ["InMarker1", "InMarker2"],
            "result_key": ["Users", "Groups"],
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_build_full_result_with_multiple_input_keys(self):
        responses = [
            {"Users": ["User1", "User2"], "Groups": ["Group1"],
             "Marker1": "m1", "Marker2": "m2"},
            {"Users": ["User3", "User4"], "Groups": ["Group2"],
             "Marker1": "m3", "Marker2": "m4"},
            {"Users": ["User5"], "Groups": ["Group3"]}
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate(
            PaginationConfig={'MaxItems': 3})
        complete = pages.build_full_result()
        expected_token = encode_token(
            {"InMarker1": "m1", "InMarker2": "m2", "boto_truncate_amount": 1})
        self.assertEqual(complete,
                         {"Users": ['User1', 'User2', 'User3'],
                          "Groups": ['Group1', 'Group2'],
                          "NextToken": expected_token})

    def test_resume_with_multiple_input_keys(self):
        responses = [
            {"Users": ["User3", "User4"], "Groups": ["Group2"],
             "Marker1": "m3", "Marker2": "m4"},
            {"Users": ["User5"], "Groups": ["Group3"]},
        ]
        self.method.side_effect = responses
        starting_token = encode_token(
            {"InMarker1": "m1", "InMarker2": "m2", "boto_truncate_amount": 1})
        pages = self.paginator.paginate(
            PaginationConfig={'MaxItems': 1,
                              'StartingToken': starting_token})
        complete = pages.build_full_result()
        expected_token = encode_token(
            {"InMarker1": "m3", "InMarker2": "m4"})
        self.assertEqual(complete,
                         {"Users": ['User4'],
                          "Groups": [],
                          "NextToken": expected_token})
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(InMarker1='m1', InMarker2='m2')])

    def test_resume_encounters_an_empty_payload(self):
        response = {"not_a_result_key": "it happens in some service"}
        self.method.return_value = response
        starting_token = encode_token(
            {"Marker": None, "boto_truncate_amount": 1})
        complete = self.paginator \
            .paginate(PaginationConfig={'StartingToken': starting_token}) \
            .build_full_result()
        self.assertEqual(complete, {})

    def test_result_key_exposed_on_paginator(self):
        self.assertEqual(
            [rk.expression for rk in self.paginator.result_keys],
            ['Users', 'Groups']
        )

    def test_result_key_exposed_on_page_iterator(self):
        pages = self.paginator.paginate(MaxItems=3)
        self.assertEqual(
            [rk.expression for rk in pages.result_keys],
            ['Users', 'Groups']
        )
class TestMultipleResultKeys(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        # This is something we'd see in s3 pagination.
        self.paginate_config = {
            "output_token": "Marker",
            "input_token": "Marker",
            "result_key": ["Users", "Groups"],
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_build_full_result_with_multiple_result_keys(self):
        responses = [
            {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"},
            {"Users": ["User2"], "Groups": ["Group2"], "Marker": "m2"},
            {"Users": ["User3"], "Groups": ["Group3"]},
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate()
        complete = pages.build_full_result()
        self.assertEqual(complete,
                         {"Users": ['User1', 'User2', 'User3'],
                          "Groups": ['Group1', 'Group2', 'Group3']})

    def test_build_full_result_with_different_length_result_keys(self):
        responses = [
            {"Users": ["User1"], "Groups": ["Group1"], "Marker": "m1"},
            # Then we stop getting "Users" output, but we get more "Groups"
            {"Users": [], "Groups": ["Group2"], "Marker": "m2"},
            {"Users": [], "Groups": ["Group3"]},
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate()
        complete = pages.build_full_result()
        self.assertEqual(complete,
                         {"Users": ['User1'],
                          "Groups": ['Group1', 'Group2', 'Group3']})

    def test_build_full_result_with_zero_length_result_key(self):
        responses = [
            # In this case the 'Users' key is always empty but we should
            # have a 'Users' key in the output, it should just have an
            # empty list for a value.
            {"Users": [], "Groups": ["Group1"], "Marker": "m1"},
            {"Users": [], "Groups": ["Group2"], "Marker": "m2"},
            {"Users": [], "Groups": ["Group3"]},
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate()
        complete = pages.build_full_result()
        self.assertEqual(complete,
                         {"Users": [],
                          "Groups": ['Group1', 'Group2', 'Group3']})

    def test_build_result_with_secondary_keys(self):
        responses = [
            {"Users": ["User1", "User2"],
             "Groups": ["Group1", "Group2"],
             "Marker": "m1"},
            {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"},
            {"Users": ["User4"], "Groups": ["Group4"]},
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate(
            PaginationConfig={'MaxItems': 1})
        complete = pages.build_full_result()
        expected_token = encode_token(
            {"Marker": None, "boto_truncate_amount": 1})
        self.assertEqual(complete,
                         {"Users": ["User1"], "Groups": ["Group1", "Group2"],
                          "NextToken": expected_token})

    def test_resume_with_secondary_keys(self):
        # This is simulating a continutation of the previous test,
        # test_build_result_with_secondary_keys.  We use the
        # token specified in the response "None___1" to continue where we
        # left off.
        responses = [
            {"Users": ["User1", "User2"],
             "Groups": ["Group1", "Group2"],
             "Marker": "m1"},
            {"Users": ["User3"], "Groups": ["Group3"], "Marker": "m2"},
            {"Users": ["User4"], "Groups": ["Group4"]},
        ]
        self.method.side_effect = responses
        starting_token = encode_token(
            {"Marker": None, "boto_truncate_amount": 1})
        pages = self.paginator.paginate(
            PaginationConfig={'MaxItems': 1,
                              'StartingToken': starting_token})
        complete = pages.build_full_result()
        # Note that the secondary keys ("Groups") are all truncated because
        # they were in the original (first) response.
        expected_token = encode_token({"Marker": "m1"})
        self.assertEqual(complete,
                         {"Users": ["User2"], "Groups": [],
                          "NextToken": expected_token})

    def test_resume_with_secondary_result_as_string(self):
        self.method.return_value = {"Users": ["User1", "User2"], "Groups": "a"}
        starting_token = encode_token(
            {"Marker": None, "boto_truncate_amount": 1})
        pages = self.paginator.paginate(
            PaginationConfig={'MaxItems': 1, 'StartingToken': starting_token})
        complete = pages.build_full_result()
        # Note that the secondary keys ("Groups") becomes empty string because
        # they were in the original (first) response.
        self.assertEqual(complete, {"Users": ["User2"], "Groups": ""})

    def test_resume_with_secondary_result_as_integer(self):
        self.method.return_value = {"Users": ["User1", "User2"], "Groups": 123}
        starting_token = encode_token(
            {"Marker": None, "boto_truncate_amount": 1})
        pages = self.paginator.paginate(
            PaginationConfig={'MaxItems': 1, 'StartingToken': starting_token})
        complete = pages.build_full_result()
        # Note that the secondary keys ("Groups") becomes zero because
        # they were in the original (first) response.
        self.assertEqual(complete, {"Users": ["User2"], "Groups": 0})
class TestKeyIterators(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        # This is something we'd see in s3 pagination.
        self.paginate_config = {
            "output_token": "Marker",
            "input_token": "Marker",
            "result_key": "Users"
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_result_key_iters(self):
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate()
        iterators = pages.result_key_iters()
        self.assertEqual(len(iterators), 1)
        self.assertEqual(list(iterators[0]),
                         ["User1", "User2", "User3"])

    def test_build_full_result_with_single_key(self):
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate()
        complete = pages.build_full_result()
        self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})

    def test_max_items_can_be_specified(self):
        paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        expected_token = encode_token({"Marker": "m1"})
        self.assertEqual(
            paginator.paginate(
                PaginationConfig={'MaxItems': 1}).build_full_result(),
            {'Users': ['User1'], 'NextToken': expected_token})

    def test_max_items_as_strings(self):
        paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        expected_token = encode_token({"Marker": "m1"})
        self.assertEqual(
            # Note MaxItems is a string here.
            paginator.paginate(
                PaginationConfig={'MaxItems': '1'}).build_full_result(),
            {'Users': ['User1'], 'NextToken': expected_token})

    def test_next_token_on_page_boundary(self):
        paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        expected_token = encode_token({"Marker": "m2"})
        self.assertEqual(
            paginator.paginate(
                PaginationConfig={'MaxItems': 2}).build_full_result(),
            {'Users': ['User1', 'User2'], 'NextToken': expected_token})

    def test_max_items_can_be_specified_truncates_response(self):
        # We're saying we only want 4 items, but notice that the second
        # page of results returns users 4-6 so we have to truncated
        # part of that second page.
        paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1", "User2", "User3"], "Marker": "m1"},
            {"Users": ["User4", "User5", "User6"], "Marker": "m2"},
            {"Users": ["User7"]},
        ]
        self.method.side_effect = responses
        expected_token = encode_token(
            {"Marker": "m1", "boto_truncate_amount": 1})
        self.assertEqual(
            paginator.paginate(
                PaginationConfig={'MaxItems': 4}).build_full_result(),
            {'Users': ['User1', 'User2', 'User3', 'User4'],
             'NextToken': expected_token})

    def test_resume_next_marker_mid_page(self):
        # This is a simulation of picking up from the response
        # from test_MaxItems_can_be_specified_truncates_response
        # We got the first 4 users, when we pick up we should get
        # User5 - User7.
        paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User4", "User5", "User6"], "Marker": "m2"},
            {"Users": ["User7"]},
        ]
        self.method.side_effect = responses
        starting_token = encode_token(
            {"Marker": "m1", "boto_truncate_amount": 1})
        pagination_config = {'StartingToken': starting_token}
        self.assertEqual(
            paginator.paginate(
                PaginationConfig=pagination_config).build_full_result(),
            {'Users': ['User5', 'User6', 'User7']})
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(Marker='m1'),
             mock.call(Marker='m2')])

    def test_max_items_exceeds_actual_amount(self):
        # Because MaxItems=10 > number of users (3), we should just return
        # all of the users.
        paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        self.assertEqual(
            paginator.paginate(
                PaginationConfig={'MaxItems': 10}).build_full_result(),
            {'Users': ['User1', 'User2', 'User3']})

    def test_bad_input_tokens(self):
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        with self.assertRaisesRegexp(ValueError, 'Bad starting token'):
            pagination_config = {'StartingToken': 'does___not___work'}
            self.paginator.paginate(
                PaginationConfig=pagination_config).build_full_result()
Example #17
0
 def paginate(self, **kwargs):
     return Paginator.paginate(self, **kwargs)
class TestPagination(unittest.TestCase):
    def setUp(self):
        self.method = mock.Mock()
        self.paginate_config = {
            'output_token': 'NextToken',
            'input_token': 'NextToken',
            'result_key': 'Foo',
        }
        self.paginator = Paginator(self.method, self.paginate_config)

    def test_result_key_available(self):
        self.assertEqual(
            [rk.expression for rk in self.paginator.result_keys],
            ['Foo']
        )

    def test_no_next_token(self):
        response = {'not_the_next_token': 'foobar'}
        self.method.return_value = response
        actual = list(self.paginator.paginate())
        self.assertEqual(actual, [{'not_the_next_token': 'foobar'}])

    def test_next_token_in_response(self):
        responses = [{'NextToken': 'token1'},
                     {'NextToken': 'token2'},
                     {'not_next_token': 'foo'}]
        self.method.side_effect = responses
        actual = list(self.paginator.paginate())
        self.assertEqual(actual, responses)
        # The first call has no next token, the second and third call should
        # have 'token1' and 'token2' respectively.
        self.assertEqual(self.method.call_args_list,
                         [mock.call(), mock.call(NextToken='token1'),
                          mock.call(NextToken='token2')])

    def test_next_token_is_string(self):
        self.paginate_config = {
            "output_token": "Marker",
            "input_token": "Marker",
            "result_key": "Users",
            "limit_key": "MaxKeys",
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]}
        ]
        self.method.side_effect = responses
        result = self.paginator.paginate(PaginationConfig={'MaxItems': 1})
        result = result.build_full_result()
        token = result.get('NextToken')
        self.assertIsInstance(token, six.string_types)

    def test_any_passed_in_args_are_unmodified(self):
        responses = [{'NextToken': 'token1'},
                     {'NextToken': 'token2'},
                     {'not_next_token': 'foo'}]
        self.method.side_effect = responses
        actual = list(self.paginator.paginate(Foo='foo', Bar='bar'))
        self.assertEqual(actual, responses)
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(Foo='foo', Bar='bar'),
             mock.call(Foo='foo', Bar='bar', NextToken='token1'),
             mock.call(Foo='foo', Bar='bar', NextToken='token2')])

    def test_exception_raised_if_same_next_token(self):
        responses = [{'NextToken': 'token1'},
                     {'NextToken': 'token2'},
                     {'NextToken': 'token2'}]
        self.method.side_effect = responses
        with self.assertRaises(PaginationError):
            list(self.paginator.paginate())

    def test_next_token_with_or_expression(self):
        self.pagination_config = {
            'output_token': 'NextToken || NextToken2',
            'input_token': 'NextToken',
            'result_key': 'Foo',
        }
        self.paginator = Paginator(self.method, self.pagination_config)
        # Verify that despite varying between NextToken and NextToken2
        # we still can extract the right next tokens.
        responses = [
            {'NextToken': 'token1'},
            {'NextToken2': 'token2'},
            # The first match found wins, so because NextToken is
            # listed before NextToken2 in the 'output_tokens' config,
            # 'token3' is chosen over 'token4'.
            {'NextToken': 'token3', 'NextToken2': 'token4'},
            {'not_next_token': 'foo'},
        ]
        self.method.side_effect = responses
        list(self.paginator.paginate())
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(),
             mock.call(NextToken='token1'),
             mock.call(NextToken='token2'),
             mock.call(NextToken='token3')])

    def test_more_tokens(self):
        # Some pagination configs have a 'more_token' key that
        # indicate whether or not the results are being paginated.
        self.paginate_config = {
            'more_results': 'IsTruncated',
            'output_token': 'NextToken',
            'input_token': 'NextToken',
            'result_key': 'Foo',
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {'Foo': [1], 'IsTruncated': True, 'NextToken': 'token1'},
            {'Foo': [2], 'IsTruncated': True, 'NextToken': 'token2'},
            {'Foo': [3], 'IsTruncated': False, 'NextToken': 'token3'},
            {'Foo': [4], 'not_next_token': 'foo'},
        ]
        self.method.side_effect = responses
        list(self.paginator.paginate())
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(),
             mock.call(NextToken='token1'),
             mock.call(NextToken='token2')])

    def test_more_tokens_is_path_expression(self):
        self.paginate_config = {
            'more_results': 'Foo.IsTruncated',
            'output_token': 'NextToken',
            'input_token': 'NextToken',
            'result_key': 'Bar',
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {'Foo': {'IsTruncated': True}, 'NextToken': 'token1'},
            {'Foo': {'IsTruncated': False}, 'NextToken': 'token2'},
        ]
        self.method.side_effect = responses
        list(self.paginator.paginate())
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(),
             mock.call(NextToken='token1')])

    def test_page_size(self):
        self.paginate_config = {
            "output_token": "Marker",
            "input_token": "Marker",
            "result_key": "Users",
            "limit_key": "MaxKeys",
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]},
        ]
        self.method.side_effect = responses
        users = []
        for page in self.paginator.paginate(PaginationConfig={'PageSize': 1}):
            users += page['Users']
        self.assertEqual(
            self.method.call_args_list,
            [mock.call(MaxKeys=1),
             mock.call(Marker='m1', MaxKeys=1),
             mock.call(Marker='m2', MaxKeys=1)]
        )

    def test_with_empty_markers(self):
        responses = [
            {"Users": ["User1"], "Marker": ""},
            {"Users": ["User1"], "Marker": ""},
            {"Users": ["User1"], "Marker": ""}
        ]
        self.method.side_effect = responses
        users = []
        for page in self.paginator.paginate():
            users += page['Users']
        # We want to stop paginating if the next token is empty.
        self.assertEqual(
            self.method.call_args_list,
            [mock.call()]
        )
        self.assertEqual(users, ['User1'])

    def test_build_full_result_with_single_key(self):
        self.paginate_config = {
            "output_token": "Marker",
            "input_token": "Marker",
            "result_key": "Users",
            "limit_key": "MaxKeys",
        }
        self.paginator = Paginator(self.method, self.paginate_config)
        responses = [
            {"Users": ["User1"], "Marker": "m1"},
            {"Users": ["User2"], "Marker": "m2"},
            {"Users": ["User3"]}
        ]
        self.method.side_effect = responses
        pages = self.paginator.paginate()
        complete = pages.build_full_result()
        self.assertEqual(complete, {'Users': ['User1', 'User2', 'User3']})