Python UrlFilterMiddleware示例

示例#1

文件： test_spidermiddleware_urlfilter.py 项目： kenzouyeh/scrapy

class TestUrlFilterMiddleware(TestCase):

    def setUp(self):
        self.spider = BaseSpider('foo')
        self.mw = UrlFilterMiddleware()

    def test_process_spider_output(self):
        res = Response('http://scrapytest.org')
        req_url = 'http://scrapytest.org/?last=1&first=2'
        reqs = [Request(req_url)]

        out = list(self.mw.process_spider_output(res, reqs, self.spider))
        self.assertEquals(out[0].url, canonicalize_url(req_url))

示例#2

文件： test_spidermiddleware_urlfilter.py 项目： kenzouyeh/scrapy

 def setUp(self):
     self.spider = BaseSpider('foo')
     self.mw = UrlFilterMiddleware()