def test_purging_keys(self):
     self.req_it = cycle(self.requests)
     ob = OverusedBuffer(self.get_once, 10, 1, 100, 10)
     ob.get_next_requests(10, overused_keys=["example.com", "www.example.com"],
                          key_type="domain")
     assert ob._get_pending_count() == 9
     ob.get_next_requests(10, overused_keys=["example.com", "www.example.com"],
                          key_type="domain") # purging of www.example.com
     assert ob._get_pending_count() == 7
Ejemplo n.º 2
0
 def test_purging_keys(self):
     self.req_it = cycle(self.requests)
     ob = OverusedBuffer(self.get_once, 10, 100)
     ob.get_next_requests(10,
                          overused_keys=["example.com", "www.example.com"],
                          key_type="domain")
     assert ob._get_pending_count() == 9
     ob.get_next_requests(10,
                          overused_keys=["example.com", "www.example.com"],
                          key_type="domain")  # purging of www.example.com
     assert ob._get_pending_count() == 7
Ejemplo n.º 3
0
    def test_base(self):
        self.req_it = iter(self.requests)
        ob = OverusedBuffer(self.get_once, 100, 10000)

        assert ob._get_pending_count() == 0
        assert set(
            ob.get_next_requests(
                10,
                overused_keys=['www.example.com', 'example1.com'],
                key_type='domain')) == set([r4, r5])
        assert ob._get_pending_count() == 4
        assert ob.get_next_requests(10,
                                    overused_keys=['www.example.com'],
                                    key_type='domain') == [r6]
        assert ob._get_pending_count() == 3

        assert ob.get_next_requests(10,
                                    overused_keys=['www.example.com'],
                                    key_type='domain') == []
        assert ob._get_pending_count() == 3

        #the max_next_requests is 3 here to cover the "len(requests) == max_next_requests" case.
        assert set(
            ob.get_next_requests(3,
                                 overused_keys=['example.com'],
                                 key_type='domain')) == set([r1, r2, r3])
        assert ob._get_pending_count() == 0

        assert ob.get_next_requests(10, overused_keys=[],
                                    key_type='domain') == []
        assert ob._get_pending_count() == 0
    def test_base(self):
        self.req_it = iter(self.requests)
        ob = OverusedBuffer(self.get_once, None, 100, None, 100)

        assert ob._get_pending_count() == 0
        assert set(ob.get_next_requests(10, overused_keys=['www.example.com', 'example1.com'],
                                        key_type='domain')) == set([r4, r5])
        assert ob._get_pending_count() == 4
        assert ob.get_next_requests(10, overused_keys=['www.example.com'],
                                    key_type='domain') == [r6]
        assert ob._get_pending_count() == 3

        assert ob.get_next_requests(10, overused_keys=['www.example.com'],
                                    key_type='domain') == []
        assert ob._get_pending_count() == 3

        #the max_next_requests is 3 here to cover the "len(requests) == max_next_requests" case.
        assert set(ob.get_next_requests(3, overused_keys=['example.com'],
                                        key_type='domain')) == set([r1, r2, r3])
        assert ob._get_pending_count() == 0

        assert ob.get_next_requests(10, overused_keys=[], key_type='domain') == []
        assert ob._get_pending_count() == 0