def _requests_to_follow(self, response): if getattr(response, "encoding", None) != None: return CrawlSpider._requests_to_follow(self, response) else: return []