Python MetaRefreshMiddleware.from_crawler Beispiele

Beispiel #1

Datei: test_downloadermiddleware_redirect.py Projekt: Prime-5/scrapy-1

 def test_ignore_tags_empty_list(self):
     crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []})
     mw = MetaRefreshMiddleware.from_crawler(crawler)
     req = Request(url='http://example.org')
     body = ('''<noscript><meta http-equiv="refresh" '''
             '''content="0;URL='http://example.org/newpage'"></noscript>''')
     rsp = HtmlResponse(req.url, body=body.encode())
     req2 = mw.process_response(req, rsp, self.spider)
     assert isinstance(req2, Request)
     self.assertEqual(req2.url, 'http://example.org/newpage')

Beispiel #2

Datei: test_downloadermiddleware_redirect.py Projekt: GennadySeleznev/Proxy-Good-Rotating

 def test_ignore_tags_1_x_list(self):
     """Test that Scrapy 1.x behavior remains possible"""
     settings = {'METAREFRESH_IGNORE_TAGS': ['script', 'noscript']}
     crawler = get_crawler(Spider, settings)
     mw = MetaRefreshMiddleware.from_crawler(crawler)
     req = Request(url='http://example.org')
     body = ('''<noscript><meta http-equiv="refresh" '''
             '''content="0;URL='http://example.org/newpage'"></noscript>''')
     rsp = HtmlResponse(req.url, body=body.encode())
     response = mw.process_response(req, rsp, self.spider)
     assert isinstance(response, Response)

Beispiel #3

Datei: test_downloadermiddleware_redirect.py Projekt: wkt2000/scrapy-1

 def setUp(self):
     crawler = get_crawler(Spider)
     self.spider = crawler._create_spider('foo')
     self.mw = MetaRefreshMiddleware.from_crawler(crawler)

Beispiel #4

Datei: test_downloadermiddleware_redirect.py Projekt: elacuesta/scrapy

 def setUp(self):
     crawler = get_crawler(Spider)
     self.spider = crawler._create_spider('foo')
     self.mw = MetaRefreshMiddleware.from_crawler(crawler)