Example #1
0
 def send(self, signal=Any, *args, **kwargs):
     sender = kwargs.pop('sender', self.sender)
     dont_log = kwargs.pop('dont_log', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
         try:
             response = robustApply(receiver,
                                    signal=signal,
                                    sender=sender,
                                    *args,
                                    **kwargs)
             if isinstance(response, defer.Deferred):
                 log.msg(
                     format=
                     'Cannot return deferreds from signal handler: %(receiver)s',
                     level=log.ERROR,
                     receiver=receiver)
         except dont_log:
             result = Failure()
         except Exception:
             result = Failure()
             log.err(result,
                     'Error caught on signal handler: %s' % receiver)
         else:
             result = response
         responses.append((receiver, result))
     return responses
Example #2
0
 def test_err_noargs(self):
     try:
         a = 1 / 0
     except:
         log.err()
     logged = self.lw.get_logged()
     self.assertIn('Traceback', logged)
     self.assertIn('ZeroDivisionError', logged)
Example #3
0
 def crawl_start_requests(self):
     # process start requests from spider
     try:
         requests = self.spider.start_requests()
         for req in arg_to_iter(requests):
             self.download(req)
     except:
         log.err(Failure(), 'Error when processing start requests.')
Example #4
0
 def crawl_start_requests(self):
     # process start requests from spider
     try:
         requests = self.spider.start_requests()
         for req in arg_to_iter(requests):
             self.download(req)
     except:
         log.err(Failure(), 'Error when processing start requests.')
Example #5
0
 def _handle_spider_error(self, failure, request):
     error = failure.value
     if isinstance(error, StopEngine):
         self.stop(error.reason)
         return
     # set `request` in a case the error was raised inside the spider
     failure.request = request
     self.signals.send(signal=signals.spider_error, failure=failure)
     if not getattr(failure.value, 'quiet', False):
         log.err(failure, 'Error when downloading %s' % request)
Example #6
0
 def _handle_spider_error(self, failure, request):
     error = failure.value
     if isinstance(error, StopEngine):
         self.stop(error.reason)
         return
     # set `request` in a case the error was raised inside the spider
     failure.request = request
     self.signals.send(signal=signals.spider_error, failure=failure)
     if not getattr(failure.value, 'quiet', False):
         log.err(failure, 'Error when downloading %s' % request)
Example #7
0
 def send(self, signal=Any, *args, **kwargs):
     sender = kwargs.pop('sender', self.sender)
     dont_log = kwargs.pop('dont_log', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
         try:
             response = robustApply(receiver, signal=signal, sender=sender,
                                    *args, **kwargs)
             if isinstance(response, defer.Deferred):
                 log.msg(format='Cannot return deferreds from signal handler: %(receiver)s',
                         level=log.ERROR, receiver=receiver)
         except dont_log:
             result = Failure()
         except Exception:
             result = Failure()
             log.err(result, 'Error caught on signal handler: %s' % receiver)
         else:
             result = response
         responses.append((receiver, result))
     return responses
Example #8
0
 def logerror(failure, recv):
     if dont_log is None or not isinstance(failure.value, dont_log):
         log.err(failure, 'Error caught on signal handler: %s' % recv)
     return failure
Example #9
0
 def test_err_failure(self):
     log.err(failure.Failure(TypeError('bad type')))
     logged = self.lw.get_logged()
     self.assertIn('Unhandled Error', logged)
     self.assertIn('TypeError', logged)
     self.assertIn('bad type', logged)
Example #10
0
 def test_err_why(self):
     log.err(TypeError('bad type'), 'Wrong type')
     logged = self.lw.get_logged(clear=False)
     self.assertEqual(self.lw.get_first_line(), '[crawlmi] ERROR: Wrong type')
     self.assertIn('TypeError', logged)
     self.assertIn('bad type', logged)
Example #11
0
 def logerror(failure, recv):
     if dont_log is None or not isinstance(failure.value, dont_log):
         log.err(failure, 'Error caught on signal handler: %s' % recv)
     return failure