def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) return dfd
def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) return dfd
def test_iter_errback_good(self): def itergood(): for x in xrange(10): yield x errors = [] out = list(iter_errback(itergood(), errors.append)) self.assertEqual(out, range(10)) self.failIf(errors)
def test_iter_errback_good(self): def itergood(): for x in range(10): yield x errors = [] out = list(iter_errback(itergood(), errors.append)) self.assertEqual(out, list(range(10))) self.assertFalse(errors)
def test_iter_errback_good(self): def itergood(): for x in xrange(10): yield x errors = [] out = list(iter_errback(itergood(), errors.append)) self.failUnlessEqual(out, range(10)) self.failIf(errors)
def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) # 这段代码,使得整个流程是往下走的,或者往下走是一个优先的趋势,很强 dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) # 感觉就是因为这一段代码,导致整个流程就需要走完某一个函数 return dfd
def test_iter_errback_bad(self): def iterbad(): for x in xrange(10): if x == 5: a = 1/0 yield x errors = [] out = list(iter_errback(iterbad(), errors.append)) self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError)
def test_iter_errback_bad(self): def iterbad(): for x in range(10): if x == 5: 1 / 0 yield x errors = [] out = list(iter_errback(iterbad(), errors.append)) self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError)
def test_iter_errback_bad(self): def iterbad(): for x in xrange(10): if x == 5: a = 1/0 yield x errors = [] out = list(iter_errback(iterbad(), errors.append)) self.failUnlessEqual(out, [0, 1, 2, 3, 4]) self.failUnlessEqual(len(errors), 1) self.failUnless(isinstance(errors[0].value, ZeroDivisionError))
def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request, response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it: Union[Generator, AsyncGenerator] if isinstance(result, AsyncIterable): it = aiter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) else: it = iter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) return dfd