def process_exception(self, request, exception, spider):
if (isinstance(exception, self.EXCEPTIONS_TO_RETRY)
and not request.meta.get('dont_retry', False)):
if isinstance(exception,
(TunnelError, defer.TimeoutError, TimeoutError)):
if self.need_switch_proxy:
request.meta["need_switch_proxy"] = True
ret = self._retry(request, exception, spider)
if ret:
return ret
else:
# 重试次数达到状态
response = HtmlResponse(url='', request=request)
response._status = 1
if isinstance(exception, Exception):
reason = global_object_name(exception.__class__)
self.logger.debug(
"max retries had reached because of {}!".format(reason))
return response
def process_exception(self, request, exception, spider):
response = HtmlResponse(url='', request=request)
response._status = 2
self.logger.debug('not contained exception: %s' % exception)
return response
