def process_response(self, request, response, spider: Spider):
if response.status == 302 or response.status == 503:
self.ERROR_COUNT += 1
print('错误次数%s' % self.ERROR_COUNT)
if self.ERROR_COUNT > 100:
spider.close(spider, 'http status error')
return response
def close(spider, reason):
if reason == 'finished':
spider.get_connector().log(spider.name, spider.ACTION_FINISHED)
else:
spider.get_connector().log(spider.name,
spider.ACTION_UNEXPECTED_END, reason)
return Spider.close(spider, reason)
def close_spider(self, spider: scrapy.Spider):
self.extract_csv(spider)
spider.close(spider, None)