Python DuplicateFilter Examples

Example #1

File: test_pipelines_duplicate_filter.py Project: dahuangfeng123/crawlmi

    def test_process_request(self):
        engine = get_engine()
        mw = DuplicateFilter(engine)

        r1 = Request('http://test.org/1')
        r2 = Request('http://test.org/2')
        r3 = Request('http://test.org/2')

        self.assertIs(mw.process_request(r1), r1)
        self.assertIs(mw.process_request(r2), r2)
        self.assertIsNone(mw.process_request(r3))

        engine.signals.send(clear_duplicate_filter)
        self.assertIs(mw.process_request(r3), r3)

Example #2

File: test_pipelines_duplicate_filter.py Project: dahuangfeng123/crawlmi

    def test_tags(self):
        engine = get_engine()
        mw = DuplicateFilter(engine)

        r1 = Request('http://test.org/', meta={'df_tag': '1'})
        r2 = Request('http://test.org/', meta={'df_tag': '2'})
        r3 = Request('http://test.org/', meta={'df_tag': '2'})

        self.assertIs(mw.process_request(r1), r1)
        self.assertIs(mw.process_request(r2), r2)
        self.assertIsNone(mw.process_request(r3))

        engine.signals.send(clear_duplicate_filter, df_tag='2')

        self.assertIsNone(mw.process_request(r1))
        self.assertIs(mw.process_request(r2), r2)
        self.assertIsNone(mw.process_request(r3))