def handle_spider_output(self, result, request, response, spider):
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, spider)
dfd = parallel(it, self.concurrent_items,
self._process_spidermw_output, request, response, spider)
return dfd
def handle_spider_output(self, result, request, response, spider):
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel(it, self.concurrent_items,
self._process_spidermw_output, request, response, spider)
return dfd
def test_iter_errback_good(self):
def itergood():
for x in xrange(10):
yield x
errors = []
out = list(iter_errback(itergood(), errors.append))
self.assertEqual(out, range(10))
self.failIf(errors)
def test_iter_errback_good(self):
def itergood():
for x in range(10):
yield x
errors = []
out = list(iter_errback(itergood(), errors.append))
self.assertEqual(out, list(range(10)))
self.assertFalse(errors)
def test_iter_errback_good(self):
def itergood():
for x in xrange(10):
yield x
errors = []
out = list(iter_errback(itergood(), errors.append))
self.failUnlessEqual(out, range(10))
self.failIf(errors)
def handle_spider_output(self, result, request, response, spider):
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, response,
spider)
# 这段代码，使得整个流程是往下走的，或者往下走是一个优先的趋势，很强
dfd = parallel(it, self.concurrent_items,
self._process_spidermw_output, request, response,
spider) # 感觉就是因为这一段代码，导致整个流程就需要走完某一个函数
return dfd
def test_iter_errback_bad(self):
def iterbad():
for x in xrange(10):
if x == 5:
a = 1/0
yield x
errors = []
out = list(iter_errback(iterbad(), errors.append))
self.assertEqual(out, [0, 1, 2, 3, 4])
self.assertEqual(len(errors), 1)
self.assertIsInstance(errors[0].value, ZeroDivisionError)
def test_iter_errback_bad(self):
def iterbad():
for x in range(10):
if x == 5:
1 / 0
yield x
errors = []
out = list(iter_errback(iterbad(), errors.append))
self.assertEqual(out, [0, 1, 2, 3, 4])
self.assertEqual(len(errors), 1)
self.assertIsInstance(errors[0].value, ZeroDivisionError)
def test_iter_errback_bad(self):
def iterbad():
for x in xrange(10):
if x == 5:
a = 1/0
yield x
errors = []
out = list(iter_errback(iterbad(), errors.append))
self.failUnlessEqual(out, [0, 1, 2, 3, 4])
self.failUnlessEqual(len(errors), 1)
self.failUnless(isinstance(errors[0].value, ZeroDivisionError))
def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
response: Response, spider: Spider) -> Deferred:
if not result:
return defer_succeed(None)
it: Union[Generator, AsyncGenerator]
if isinstance(result, AsyncIterable):
it = aiter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
else:
it = iter_errback(result, self.handle_spider_error, request, response, spider)
dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
request, response, spider)
return dfd
