def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) return dfd
def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) return dfd
def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) # 这段代码,使得整个流程是往下走的,或者往下走是一个优先的趋势,很强 dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) # 感觉就是因为这一段代码,导致整个流程就需要走完某一个函数 return dfd
def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request, response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it: Union[Generator, AsyncGenerator] if isinstance(result, AsyncIterable): it = aiter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) else: it = iter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) return dfd