def send(self, signal=Any, *args, **kwargs):
sender = kwargs.pop('sender', self.sender)
dont_log = kwargs.pop('dont_log', None)
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
try:
response = robustApply(receiver,
signal=signal,
sender=sender,
*args,
**kwargs)
if isinstance(response, defer.Deferred):
log.msg(
format=
'Cannot return deferreds from signal handler: %(receiver)s',
level=log.ERROR,
receiver=receiver)
except dont_log:
result = Failure()
except Exception:
result = Failure()
log.err(result,
'Error caught on signal handler: %s' % receiver)
else:
result = response
responses.append((receiver, result))
return responses
def test_err_noargs(self):
try:
a = 1 / 0
except:
log.err()
logged = self.lw.get_logged()
self.assertIn('Traceback', logged)
self.assertIn('ZeroDivisionError', logged)
def crawl_start_requests(self):
# process start requests from spider
try:
requests = self.spider.start_requests()
for req in arg_to_iter(requests):
self.download(req)
except:
log.err(Failure(), 'Error when processing start requests.')
def crawl_start_requests(self):
# process start requests from spider
try:
requests = self.spider.start_requests()
for req in arg_to_iter(requests):
self.download(req)
except:
log.err(Failure(), 'Error when processing start requests.')
def _handle_spider_error(self, failure, request):
error = failure.value
if isinstance(error, StopEngine):
self.stop(error.reason)
return
# set `request` in a case the error was raised inside the spider
failure.request = request
self.signals.send(signal=signals.spider_error, failure=failure)
if not getattr(failure.value, 'quiet', False):
log.err(failure, 'Error when downloading %s' % request)
def _handle_spider_error(self, failure, request):
error = failure.value
if isinstance(error, StopEngine):
self.stop(error.reason)
return
# set `request` in a case the error was raised inside the spider
failure.request = request
self.signals.send(signal=signals.spider_error, failure=failure)
if not getattr(failure.value, 'quiet', False):
log.err(failure, 'Error when downloading %s' % request)
def send(self, signal=Any, *args, **kwargs):
sender = kwargs.pop('sender', self.sender)
dont_log = kwargs.pop('dont_log', None)
responses = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
try:
response = robustApply(receiver, signal=signal, sender=sender,
*args, **kwargs)
if isinstance(response, defer.Deferred):
log.msg(format='Cannot return deferreds from signal handler: %(receiver)s',
level=log.ERROR, receiver=receiver)
except dont_log:
result = Failure()
except Exception:
result = Failure()
log.err(result, 'Error caught on signal handler: %s' % receiver)
else:
result = response
responses.append((receiver, result))
return responses
def logerror(failure, recv):
if dont_log is None or not isinstance(failure.value, dont_log):
log.err(failure, 'Error caught on signal handler: %s' % recv)
return failure
def test_err_failure(self):
log.err(failure.Failure(TypeError('bad type')))
logged = self.lw.get_logged()
self.assertIn('Unhandled Error', logged)
self.assertIn('TypeError', logged)
self.assertIn('bad type', logged)
def test_err_why(self):
log.err(TypeError('bad type'), 'Wrong type')
logged = self.lw.get_logged(clear=False)
self.assertEqual(self.lw.get_first_line(), '[crawlmi] ERROR: Wrong type')
self.assertIn('TypeError', logged)
self.assertIn('bad type', logged)
def logerror(failure, recv):
if dont_log is None or not isinstance(failure.value, dont_log):
log.err(failure, 'Error caught on signal handler: %s' % recv)
return failure