def setUp(self): self.spider = BaseSpider('scrapytest.org') self.mw = DownloaderStats() stats.open_spider(self.spider) self.req = Request('http://scrapytest.org') self.res = Response('scrapytest.org', status=400)
def open_spider(self, spider): assert self.has_capacity(), "No free spider slots when opening %r" % spider.name log.msg("Spider opened", spider=spider) yield self.scheduler.open_spider(spider) self.downloader.open_spider(spider) yield self.scraper.open_spider(spider) stats.open_spider(spider) yield send_catch_log_deferred(signals.spider_opened, spider=spider) self.next_request(spider)
def open_spider(self, spider): log.msg("Spider opened", spider=spider) self.next_request(spider) self.downloader.open_spider(spider) self.scraper.open_spider(spider) stats.open_spider(spider) send_catch_log(signals.spider_opened, sender=self.__class__, spider=spider)
def open_spider(self, spider): assert self.has_capacity(), "No free spider slots when opening %r" % \ spider.name log.msg("Spider opened", spider=spider) yield self.scheduler.open_spider(spider) self.downloader.open_spider(spider) yield self.scraper.open_spider(spider) stats.open_spider(spider) yield send_catch_log_deferred(signals.spider_opened, spider=spider) self.next_request(spider)
def setUp(self): settings.disabled = False settings.overrides['DEPTH_LIMIT'] = 1 settings.overrides['DEPTH_STATS'] = True self.spider = BaseSpider('scrapytest.org') stats.open_spider(self.spider) self.mw = DepthMiddleware() self.assertEquals(stats.get_value('envinfo/request_depth_limit'), 1)
def open_spider(self, spider, start_requests=None, close_if_idle=True): assert self.has_capacity(), "No free spider slots when opening %r" % spider.name log.msg("Spider opened", spider=spider) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_settings(self.settings) slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler) self.slots[spider] = slot yield scheduler.open(spider) yield self.scraper.open_spider(spider) stats.open_spider(spider) yield send_catch_log_deferred(signals.spider_opened, spider=spider) slot.nextcall.schedule()
def open_spider(self, spider, start_requests=None, close_if_idle=True): assert self.has_capacity(), "No free spider slots when opening %r" % \ spider.name log.msg("Spider opened", spider=spider) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_settings(self.settings) slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler) self.slots[spider] = slot yield scheduler.open(spider) yield self.scraper.open_spider(spider) stats.open_spider(spider) yield send_catch_log_deferred(signals.spider_opened, spider=spider) slot.nextcall.schedule()
def setUp(self): self.spider = BaseSpider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', headers={'User-Agent': 'test'}) self.response = Response('http://www.example.com', headers={'Content-Type': 'text/html'}, body='test body', status=202) stats.open_spider(self.spider)