Python RFPDupeFilter.close Examples

Example #1

File: test_dupefilter.py Project: OriginalCoder0/Python

class TestRFPDupeFilter(object):
    def setup(self):
        self.server = get_redis_mock()
        self.key = 'dupefilter:1'
        self.df = RFPDupeFilter(self.server, self.key)

    def test_request_seen(self):
        req = Request('http://example.com')
        assert not self.df.request_seen(req)
        assert self.df.request_seen(req)

    def test_overridable_request_fingerprinter(self):
        req = Request('http://example.com')
        self.df.request_fingerprint = mock.Mock(
            wraps=self.df.request_fingerprint)
        assert not self.df.request_seen(req)
        self.df.request_fingerprint.assert_called_with(req)

    def test_clear_deletes(self):
        self.df.clear()
        self.server.delete.assert_called_with(self.key)

    def test_close_calls_clear(self):
        self.df.clear = mock.Mock(wraps=self.df.clear)
        self.df.close()
        self.df.close(reason='foo')
        assert self.df.clear.call_count == 2

Example #2

File: test_dupefilter.py Project: lnlantian/Scrapy

class TestRFPDupeFilter(object):

    def setup(self):
        self.server = get_redis_mock()
        self.key = 'dupefilter:1'
        self.df = RFPDupeFilter(self.server, self.key)

    def test_request_seen(self):
        req = Request('http://example.com')
        assert not self.df.request_seen(req)
        assert self.df.request_seen(req)

    def test_overridable_request_fingerprinter(self):
        req = Request('http://example.com')
        self.df.request_fingerprint = mock.Mock(wraps=self.df.request_fingerprint)
        assert not self.df.request_seen(req)
        self.df.request_fingerprint.assert_called_with(req)

    def test_clear_deletes(self):
        self.df.clear()
        self.server.delete.assert_called_with(self.key)

    def test_close_calls_clear(self):
        self.df.clear = mock.Mock(wraps=self.df.clear)
        self.df.close()
        self.df.close(reason='foo')
        assert self.df.clear.call_count == 2

Example #3

File: test_scrapy_redis.py Project: OriginalCoder0/Python

class DupeFilterTest(RedisTestMixin, TestCase):
    def setUp(self):
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.clear_keys(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')

Example #4

class DupeFilterTest(TestCase):
    def setUp(self):
        self.server = redis.Redis(REDIS_HOST, REDIS_PORT)
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.server.delete(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')

Example #5

File: test_scrapy_redis.py Project: 15310944349/scrapy-redis

class DupeFilterTest(RedisTestMixin, TestCase):

    def setUp(self):
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.clear_keys(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')

Example #6

File: allPythonContent.py Project: Mondego/pyreco

class DupeFilterTest(TestCase):

    def setUp(self):
        self.server = redis.Redis(REDIS_HOST, REDIS_PORT)
        self.key = 'scrapy_redis:tests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key)

    def tearDown(self):
        self.server.delete(self.key)

    def test_dupe_filter(self):
        req = Request('http://example.com')

        self.assertFalse(self.df.request_seen(req))
        self.assertTrue(self.df.request_seen(req))

        self.df.close('nothing')

Example #7

File: Utility.py Project: actlea/MyProj

class DupeFilterTest():   

    def __init__(self):
        self.setUp()
        
    def setUp(self):
        self.server = redis.Redis(REDIS_HOST, REDIS_PORT)
        self.key = 'Fcrawler:requests:dupefilter:'
        self.df = RFPDupeFilter(self.server, self.key) 
        
    def tearDown(self):
        self.server.delete(self.key)

    def request_dupe_filter(self, url):
        req = Request(url)      
        flag = self.df.request_seen(req)

        self.df.close('nothing')
        
        return flag