def get_defaults_spider_mw(self): crawler = get_crawler() spider = BaseSpider('foo') spider.set_crawler(crawler) defaults = dict([(k, [v]) for k, v in \ crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) return defaults, spider, DefaultHeadersMiddleware()
def get_defaults_spider_mw(self): crawler = get_crawler() spider = BaseSpider('foo') spider.set_crawler(crawler) defaults = dict([(k, [v]) for k, v in \ crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) return defaults, spider, DefaultHeadersMiddleware()
class ManagerTestCase(TestCase): settings_dict = None def setUp(self): self.crawler = get_crawler(self.settings_dict) self.spider = BaseSpider('foo') self.spider.set_crawler(self.crawler) self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) # some mw depends on stats collector self.crawler.stats.open_spider(self.spider) return self.mwman.open_spider(self.spider) def tearDown(self): self.crawler.stats.close_spider(self.spider, '') return self.mwman.close_spider(self.spider) def _download(self, request, response=None): """Executes downloader mw manager's download method and returns the result (Request or Response) or raise exception in case of failure. """ if not response: response = Response(request.url) def download_func(**kwargs): return response dfd = self.mwman.download(download_func, request, self.spider) # catch deferred result and return the value results = [] dfd.addBoth(results.append) self._wait(dfd) ret = results[0] if isinstance(ret, Failure): ret.raiseException() return ret
class ManagerTestCase(TestCase): settings_dict = None def setUp(self): self.crawler = get_crawler(self.settings_dict) self.spider = BaseSpider('foo') self.spider.set_crawler(self.crawler) self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) # some mw depends on stats collector self.crawler.stats.open_spider(self.spider) return self.mwman.open_spider(self.spider) def tearDown(self): self.crawler.stats.close_spider(self.spider, '') return self.mwman.close_spider(self.spider) def _download(self, request, response=None): """Executes downloader mw manager's download method and returns the result (Request or Response) or raise exception in case of failure. """ if not response: response = Response(request.url) def download_func(**kwargs): return response dfd = self.mwman.download(download_func, request, self.spider) # catch deferred result and return the value results = [] dfd.addBoth(results.append) self._wait(dfd) ret = results[0] if isinstance(ret, Failure): ret.raiseException() return ret
def get_spider_and_mw(self, default_useragent): crawler = get_crawler({'USER_AGENT': default_useragent}) spider = BaseSpider('foo') spider.set_crawler(crawler) return spider, UserAgentMiddleware()
def get_request_spider_mw(self): crawler = get_crawler() spider = BaseSpider('foo') spider.set_crawler(crawler) request = Request('http://scrapytest.org/') return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
def get_spider_and_mw(self, default_useragent): crawler = get_crawler({'USER_AGENT': default_useragent}) spider = BaseSpider('foo') spider.set_crawler(crawler) return spider, UserAgentMiddleware.from_crawler(crawler)
def get_request_spider_mw(self): crawler = get_crawler() spider = BaseSpider('foo') spider.set_crawler(crawler) request = Request('http://scrapytest.org/') return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)