def errback_twisted(self, failure):
if failure.check(TimeoutError, TCPTimedOutError, DNSLookupError):
pass
if failure.check(HttpError):
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.response
logger.error('HttpError on %s', response.url)
def errback_twisted(self, failure):
if failure.check(TimeoutError, TCPTimedOutError, DNSLookupError):
request = failure.request
connect.sadd(fail_url, request.url)
if failure.check(HttpError):
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
request = failure.value.request
connect.sadd(fail_url, request.url)
logger.error('HttpError on %s', response.url)
def errback_twisted(self, failure):
if failure.check(TimeoutError, TCPTimedOutError, DNSLookupError):
while True:
self.connect.blpop(ip_cookie_key, 1)
if self.connect.llen(ip_cookie_key) == 0:
break
IPCookie().get_cookies()
if failure.check(HttpError):
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
while True:
self.connect.blpop(ip_cookie_key, 1)
if self.connect.llen(ip_cookie_key) == 0:
break
IPCookie().get_cookies()
response = failure.response
logger.error('HttpError on %s', response.url)
