Esempio n. 1
0
 def process_response(self, request, response, spider: Spider):
     if response.status == 302 or response.status == 503:
         self.ERROR_COUNT += 1
         print('错误次数%s' % self.ERROR_COUNT)
         if self.ERROR_COUNT > 100:
             spider.close(spider, 'http status error')
     return response
Esempio n. 2
0
 def close(spider, reason):
     if reason == 'finished':
         spider.get_connector().log(spider.name, spider.ACTION_FINISHED)
     else:
         spider.get_connector().log(spider.name,
                                    spider.ACTION_UNEXPECTED_END, reason)
     return Spider.close(spider, reason)
Esempio n. 3
0
 def close_spider(self, spider: scrapy.Spider):
     self.extract_csv(spider)
     spider.close(spider, None)