Python DefaultHeadersMiddleware Examples

Example #1

File: test_downloadermiddleware_defaultheaders.py Project: richard-ma/CodeReading

 def get_defaults_spider_mw(self):
     crawler = get_crawler()
     spider = BaseSpider('foo')
     spider.set_crawler(crawler)
     defaults = dict([(k, [v]) for k, v in \
         crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])
     return defaults, spider, DefaultHeadersMiddleware()

Example #2

File: test_downloadermiddleware_defaultheaders.py Project: 00gpowe/scrapy

 def get_defaults_spider_mw(self):
     crawler = get_crawler()
     spider = BaseSpider('foo')
     spider.set_crawler(crawler)
     defaults = dict([(k, [v]) for k, v in \
         crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])
     return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)

Example #3

File: test_downloadermiddleware_defaultheaders.py Project: kenzouyeh/scrapy

class TestDefaultHeadersMiddleware(TestCase):

    def setUp(self):
        self.spider = BaseSpider('foo')
        self.mw = DefaultHeadersMiddleware()
        self.default_headers = dict([(k, [v]) for k, v in \
            settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])

    def test_process_request(self):
        req = Request('http://www.scrapytest.org')
        self.mw.process_request(req, self.spider)
        self.assertEquals(req.headers, self.default_headers)

    def test_update_headers(self):
        headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
        req = Request('http://www.scrapytest.org', headers=headers)
        self.assertEquals(req.headers, headers)

        self.mw.process_request(req, self.spider)
        self.default_headers.update(headers)
        self.assertEquals(req.headers, self.default_headers)

Example #4

File: test_downloadermiddleware_defaultheaders.py Project: kenzouyeh/scrapy

 def setUp(self):
     self.spider = BaseSpider('foo')
     self.mw = DefaultHeadersMiddleware()
     self.default_headers = dict([(k, [v]) for k, v in \
         settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])

Example #5

File: test_downloadermiddleware_defaultheaders.py Project: zxsted/scrapy

 def get_defaults_spider_mw(self):
     crawler = get_crawler(Spider)
     spider = crawler._create_spider('foo')
     defaults = dict([(k, [v]) for k, v in \
         six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))])
     return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)