Python HttpCacheMiddleware示例

示例#1

文件： test_downloadermiddleware_httpcache.py 项目： frrp/scrapy

 def test_middleware(self):
     mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
     assert mw.process_request(self.request, self.spider) is None
     mw.process_response(self.request, self.response, self.spider)
     response = mw.process_request(self.request, self.spider)
     assert isinstance(response, HtmlResponse)
     self.assertEqualResponse(self.response, response)
     assert 'cached' in response.flags

示例#2

文件： test_downloadermiddleware_httpcache.py 项目： pyarnold/scrapy

 def _middleware(self, **new_settings):
     settings = self._get_settings(**new_settings)
     mw = HttpCacheMiddleware(settings, self.crawler.stats)
     mw.spider_opened(self.spider)
     try:
         yield mw
     finally:
         mw.spider_closed(self.spider)

示例#3

文件： test_downloadermiddleware_httpcache.py 项目： frrp/scrapy

 def test_different_request_response_urls(self):
     mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
     req = Request('http://host.com/path')
     res = Response('http://host2.net/test.html')
     assert mw.process_request(req, self.spider) is None
     mw.process_response(req, res, self.spider)
     cached = mw.process_request(req, self.spider)
     assert isinstance(cached, Response)
     self.assertEqualResponse(res, cached)
     assert 'cached' in cached.flags

示例#4

 def _middleware(self, **new_settings):
     settings = self._get_settings(**new_settings)
     mw = HttpCacheMiddleware(settings, self.crawler.stats)
     mw.spider_opened(self.spider)
     try:
         yield mw
     finally:
         mw.spider_closed(self.spider)

示例#5

 def test_middleware(self):
     mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
     assert mw.process_request(self.request, self.spider) is None
     mw.process_response(self.request, self.response, self.spider)
     response = mw.process_request(self.request, self.spider)
     assert isinstance(response, HtmlResponse)
     self.assertEqualResponse(self.response, response)
     assert 'cached' in response.flags

示例#6

 def test_different_request_response_urls(self):
     mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
     req = Request('http://host.com/path')
     res = Response('http://host2.net/test.html')
     assert mw.process_request(req, self.spider) is None
     mw.process_response(req, res, self.spider)
     cached = mw.process_request(req, self.spider)
     assert isinstance(cached, Response)
     self.assertEqualResponse(res, cached)
     assert 'cached' in cached.flags

示例#7

文件： test_downloadermiddleware_httpcache.py 项目： AnthonyZhu/scrapy

 def _get_middleware(self, **new_settings):
     mw = HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats)
     mw.spider_opened(self.spider)
     return mw

示例#8

 def _get_middleware(self, **new_settings):
     return HttpCacheMiddleware(self._get_settings(**new_settings),
                                self.crawler.stats)

示例#9

文件： test_downloadermiddleware_httpcache.py 项目： weisbeck/403Section

 def _get_middleware(self, **new_settings):
     mw = HttpCacheMiddleware(self._get_settings(**new_settings),
                              self.crawler.stats)
     mw.spider_opened(self.spider)
     return mw

示例#10

 def _get_middleware(self, **new_settings):
     return HttpCacheMiddleware(self._get_settings(**new_settings))