def errback_twisted(self, failure): if failure.check(TimeoutError, TCPTimedOutError, DNSLookupError): pass if failure.check(HttpError): # these exceptions come from HttpError spider middleware # you can get the non-200 response response = failure.response logger.error('HttpError on %s', response.url)
def errback_twisted(self, failure): if failure.check(TimeoutError, TCPTimedOutError, DNSLookupError): request = failure.request connect.sadd(fail_url, request.url) if failure.check(HttpError): # these exceptions come from HttpError spider middleware # you can get the non-200 response response = failure.value.response request = failure.value.request connect.sadd(fail_url, request.url) logger.error('HttpError on %s', response.url)
def errback_twisted(self, failure): if failure.check(TimeoutError, TCPTimedOutError, DNSLookupError): while True: self.connect.blpop(ip_cookie_key, 1) if self.connect.llen(ip_cookie_key) == 0: break IPCookie().get_cookies() if failure.check(HttpError): # these exceptions come from HttpError spider middleware # you can get the non-200 response while True: self.connect.blpop(ip_cookie_key, 1) if self.connect.llen(ip_cookie_key) == 0: break IPCookie().get_cookies() response = failure.response logger.error('HttpError on %s', response.url)