Python close_spider 예제들

예제 #1

    def close_spider(self, spider, reason='cancelled'):
        """Close (cancel) spider and clear all its outstanding requests"""

        slot = self.slots[spider]
        if slot.closing:
            return slot.closing
        log.msg("Closing spider (%s)" % reason, spider=spider)

        dfd = slot.close()

        dfd.addBoth(lambda _: self.scraper.close_spider(spider))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: slot.scheduler.close(reason))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: send_catch_log_deferred(signal=signals.spider_closed, \
            spider=spider, reason=reason))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))

        dfd.addBoth(lambda _: self.slots.pop(spider))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: self._spider_closed_callback(spider))

        return dfd

예제 #2

파일: engine.py 프로젝트: tml/scrapy

    def close_spider(self, spider, reason='cancelled'):
        """Close (cancel) spider and clear all its outstanding requests"""

        slot = self.slots[spider]
        if slot.closing:
            return slot.closing
        log.msg("Closing spider (%s)" % reason, spider=spider)

        dfd = slot.close()

        dfd.addBoth(lambda _: self.scraper.close_spider(spider))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: slot.scheduler.close(reason))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: send_catch_log_deferred(signal=signals.spider_closed, \
            spider=spider, reason=reason))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))

        dfd.addBoth(lambda _: self.slots.pop(spider))
        dfd.addErrback(log.err, spider=spider)

        dfd.addBoth(lambda _: self._spider_closed_callback(spider))

        return dfd

예제 #3

파일: engine.py 프로젝트: serkanh/scrapy

 def _finish_closing_spider(self, spider):
     """This function is called after the spider has been closed"""
     self.scheduler.close_spider(spider)
     self.scraper.close_spider(spider)
     reason = self.closing.pop(spider, 'finished')
     send_catch_log(signal=signals.spider_closed, sender=self.__class__, \
         spider=spider, reason=reason)
     stats.close_spider(spider, reason=reason)
     call = self._next_request_calls.pop(spider, None)
     if call and call.active():
         call.cancel()
     dfd = defer.maybeDeferred(spiders.close_spider, spider)
     dfd.addErrback(log.err, "Unhandled error on SpiderManager.close_spider()",
         spider=spider)
     dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))
     reactor.callLater(0, self._mainloop)
     return dfd

예제 #4

파일: engine.py 프로젝트: zhangcheng/scrapy

 def _finish_closing_spider(self, spider):
     """This function is called after the spider has been closed"""
     reason = self.closing.pop(spider, "finished")
     call = self._next_request_calls.pop(spider, None)
     if call and call.active():
         call.cancel()
     dfd = send_catch_log_deferred(signal=signals.spider_closed, spider=spider, reason=reason)
     dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
     dfd.addErrback(log.err, "Unhandled error in stats.close_spider()", spider=spider)
     dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))
     dfd.addBoth(lambda _: self.closing_dfds.pop(spider).callback(spider))
     dfd.addBoth(lambda _: self._spider_closed_callback(spider))
     return dfd

예제 #5

파일: engine.py 프로젝트: richard-ma/CodeReading

 def _finish_closing_spider(self, spider):
     """This function is called after the spider has been closed"""
     reason = self.closing.pop(spider, 'finished')
     call = self._next_request_calls.pop(spider, None)
     if call and call.active():
         call.cancel()
     dfd = send_catch_log_deferred(signal=signals.spider_closed, \
         spider=spider, reason=reason)
     dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
     dfd.addErrback(log.err,
                    "Unhandled error in stats.close_spider()",
                    spider=spider)
     dfd.addBoth(
         lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))
     dfd.addBoth(lambda _: self.closing_dfds.pop(spider).callback(spider))
     dfd.addBoth(lambda _: self._spider_closed_callback(spider))
     return dfd

예제 #6

파일: test_downloadermiddleware_stats.py 프로젝트: Aaron1011/oh-mainline

 def tearDown(self):
     stats.close_spider(self.spider, '')

예제 #7

파일: test_downloadermiddleware_stats.py 프로젝트: reenvs/self-summary

 def tearDown(self):
     stats.close_spider(self.spider, '')

예제 #8

파일: test_spidermiddleware_depth.py 프로젝트: kenzouyeh/scrapy

    def tearDown(self):
        del settings.overrides['DEPTH_LIMIT']
        del settings.overrides['DEPTH_STATS']
        settings.disabled = True

        stats.close_spider(self.spider, '')

예제 #9

 def tearDown(self):
     stats.close_spider(self.spider, '')
     shutil.rmtree(self.tmpdir)