Example #1
0
class TestRFPDupeFilter(object):
    def setup(self):
        self.server = get_redis_mock()
        self.key = 'dupefilter:1'
        self.df = RFPDupeFilter(self.server, self.key)

    def test_request_seen(self):
        req = Request('http://example.com')
        assert not self.df.request_seen(req)
        assert self.df.request_seen(req)

    def test_overridable_request_fingerprinter(self):
        req = Request('http://example.com')
        self.df.request_fingerprint = mock.Mock(
            wraps=self.df.request_fingerprint)
        assert not self.df.request_seen(req)
        self.df.request_fingerprint.assert_called_with(req)

    def test_clear_deletes(self):
        self.df.clear()
        self.server.delete.assert_called_with(self.key)

    def test_close_calls_clear(self):
        self.df.clear = mock.Mock(wraps=self.df.clear)
        self.df.close()
        self.df.close(reason='foo')
        assert self.df.clear.call_count == 2
Example #2
0
class TestRFPDupeFilter(object):

    def setup(self):
        self.server = get_redis_mock()
        self.key = 'dupefilter:1'
        self.df = RFPDupeFilter(self.server, self.key)

    def test_request_seen(self):
        req = Request('http://example.com')
        assert not self.df.request_seen(req)
        assert self.df.request_seen(req)

    def test_overridable_request_fingerprinter(self):
        req = Request('http://example.com')
        self.df.request_fingerprint = mock.Mock(wraps=self.df.request_fingerprint)
        assert not self.df.request_seen(req)
        self.df.request_fingerprint.assert_called_with(req)

    def test_clear_deletes(self):
        self.df.clear()
        self.server.delete.assert_called_with(self.key)

    def test_close_calls_clear(self):
        self.df.clear = mock.Mock(wraps=self.df.clear)
        self.df.close()
        self.df.close(reason='foo')
        assert self.df.clear.call_count == 2
class DupeFilterTest(RedisTestMixin, TestCase):
    def setUp(self):
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.clear_keys(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')
Example #4
0
class DupeFilterTest(TestCase):
    def setUp(self):
        self.server = redis.Redis(REDIS_HOST, REDIS_PORT)
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.server.delete(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')
class DupeFilterTest(RedisTestMixin, TestCase):

    def setUp(self):
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.clear_keys(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')
Example #6
0
class DupeFilterTest(TestCase):

    def setUp(self):
        self.server = redis.Redis(REDIS_HOST, REDIS_PORT)
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.server.delete(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')
Example #7
0
class DupeFilterTest():   

    def __init__(self):
        self.setUp()
        
    def setUp(self):
        self.server = redis.Redis(REDIS_HOST, REDIS_PORT)
        self.key = 'Fcrawler:requests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key) 
        
    def tearDown(self):
        self.server.delete(self.key)

    def request_dupe_filter(self, url):
        req = Request(url)      
        flag = self.df.request_seen(req)

        self.df.close('nothing')
        
        return flag