def setUp(self):
        self.spider = BaseSpider('scrapytest.org')
        self.mw = DownloaderStats()

        stats.open_spider(self.spider)

        self.req = Request('http://scrapytest.org')
        self.res = Response('scrapytest.org', status=400)
    def setUp(self):
        self.spider = BaseSpider('scrapytest.org')
        self.mw = DownloaderStats()

        stats.open_spider(self.spider)

        self.req = Request('http://scrapytest.org')
        self.res = Response('scrapytest.org', status=400)
Exemplo n.º 3
0
 def open_spider(self, spider):
     assert self.has_capacity(), "No free spider slots when opening %r" % spider.name
     log.msg("Spider opened", spider=spider)
     yield self.scheduler.open_spider(spider)
     self.downloader.open_spider(spider)
     yield self.scraper.open_spider(spider)
     stats.open_spider(spider)
     yield send_catch_log_deferred(signals.spider_opened, spider=spider)
     self.next_request(spider)
Exemplo n.º 4
0
    def open_spider(self, spider):
        log.msg("Spider opened", spider=spider)
        self.next_request(spider)

        self.downloader.open_spider(spider)
        self.scraper.open_spider(spider)
        stats.open_spider(spider)

        send_catch_log(signals.spider_opened, sender=self.__class__, spider=spider)
Exemplo n.º 5
0
 def open_spider(self, spider):
     assert self.has_capacity(), "No free spider slots when opening %r" % \
         spider.name
     log.msg("Spider opened", spider=spider)
     yield self.scheduler.open_spider(spider)
     self.downloader.open_spider(spider)
     yield self.scraper.open_spider(spider)
     stats.open_spider(spider)
     yield send_catch_log_deferred(signals.spider_opened, spider=spider)
     self.next_request(spider)
    def setUp(self):
        settings.disabled = False
        settings.overrides['DEPTH_LIMIT'] = 1
        settings.overrides['DEPTH_STATS'] = True

        self.spider = BaseSpider('scrapytest.org')

        stats.open_spider(self.spider)

        self.mw = DepthMiddleware()
        self.assertEquals(stats.get_value('envinfo/request_depth_limit'), 1)
Exemplo n.º 7
0
 def open_spider(self, spider, start_requests=None, close_if_idle=True):
     assert self.has_capacity(), "No free spider slots when opening %r" % spider.name
     log.msg("Spider opened", spider=spider)
     nextcall = CallLaterOnce(self._next_request, spider)
     scheduler = self.scheduler_cls.from_settings(self.settings)
     slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler)
     self.slots[spider] = slot
     yield scheduler.open(spider)
     yield self.scraper.open_spider(spider)
     stats.open_spider(spider)
     yield send_catch_log_deferred(signals.spider_opened, spider=spider)
     slot.nextcall.schedule()
Exemplo n.º 8
0
 def open_spider(self, spider, start_requests=None, close_if_idle=True):
     assert self.has_capacity(), "No free spider slots when opening %r" % \
         spider.name
     log.msg("Spider opened", spider=spider)
     nextcall = CallLaterOnce(self._next_request, spider)
     scheduler = self.scheduler_cls.from_settings(self.settings)
     slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler)
     self.slots[spider] = slot
     yield scheduler.open(spider)
     yield self.scraper.open_spider(spider)
     stats.open_spider(spider)
     yield send_catch_log_deferred(signals.spider_opened, spider=spider)
     slot.nextcall.schedule()
Exemplo n.º 9
0
 def setUp(self):
     self.spider = BaseSpider('example.com')
     self.tmpdir = tempfile.mkdtemp()
     self.request = Request('http://www.example.com', headers={'User-Agent': 'test'})
     self.response = Response('http://www.example.com', headers={'Content-Type': 'text/html'}, body='test body', status=202)
     stats.open_spider(self.spider)