def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) callback = request.callback or spider.parse warn_on_generator_with_return_value(spider, callback) warn_on_generator_with_return_value(spider, request.errback) dfd.addCallbacks(callback=callback, errback=request.errback, callbackKeywords=request.cb_kwargs) return dfd.addCallback(iterate_spider_output)
def call_spider(self, result, request, spider): if isinstance(result, Response): callback = request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) result.request = request return iterate_spider_output(callback(result)) elif request.errback is not None: warn_on_generator_with_return_value(spider, request.errback) return iterate_spider_output(request.errback(result)) else: return result
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request callback = result.request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs) else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) dfd = defer_fail(result) dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output)
def call_spider(self, result, request, spider): if isinstance(result, Response): #从spider中拿到的事Response对象 if getattr(result, "request", None) is None: result.request = request callback = result.request.callback or spider._parse # 从request对象里面拿到 对应的callback 否则传入spider的_parse函数作为callback warn_on_generator_with_return_value(spider, callback) dfd = defer_succeed(result) dfd.addCallback(callback, **result.request.cb_kwargs ) # 将spider的callback 添加到 deferred的回调链路上 else: # result is a Failure result.request = request warn_on_generator_with_return_value(spider, request.errback) dfd = defer_fail(result) dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output)
def test_generators_return_something(self): def f1(): yield 1 return 2 def g1(): yield 1 return "asdf" def h1(): yield 1 def helper(): return 0 yield helper() return 2 def i1(): """ docstring """ url = """ https://example.org """ yield url return 1 assert is_generator_with_return_value(top_level_return_something) assert is_generator_with_return_value(f1) assert is_generator_with_return_value(g1) assert is_generator_with_return_value(h1) assert is_generator_with_return_value(i1) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_something) self.assertEqual(len(w), 1) self.assertIn( 'The "NoneType.top_level_return_something" method is a generator', str(w[0].message)) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f1) self.assertEqual(len(w), 1) self.assertIn('The "NoneType.f1" method is a generator', str(w[0].message)) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g1) self.assertEqual(len(w), 1) self.assertIn('The "NoneType.g1" method is a generator', str(w[0].message)) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h1) self.assertEqual(len(w), 1) self.assertIn('The "NoneType.h1" method is a generator', str(w[0].message)) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i1) self.assertEqual(len(w), 1) self.assertIn('The "NoneType.i1" method is a generator', str(w[0].message))
def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) self.assertEqual(len(w), 1) self.assertIn('Unable to determine', str(w[0].message))
def test_generators_return_none(self): def f2(): yield 1 return None def g2(): yield 1 return def h2(): yield 1 def i2(): yield 1 yield from generator_that_returns_stuff() def j2(): yield 1 def helper(): return 0 yield helper() def k2(): """ docstring """ url = """ https://example.org """ yield url return def l2(): return assert not is_generator_with_return_value(top_level_return_none) assert not is_generator_with_return_value(f2) assert not is_generator_with_return_value(g2) assert not is_generator_with_return_value(h2) assert not is_generator_with_return_value(i2) assert not is_generator_with_return_value(j2) # not recursive assert not is_generator_with_return_value(k2) # not recursive assert not is_generator_with_return_value(l2) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, top_level_return_none) self.assertEqual(len(w), 0) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, f2) self.assertEqual(len(w), 0) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, g2) self.assertEqual(len(w), 0) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, h2) self.assertEqual(len(w), 0) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, i2) self.assertEqual(len(w), 0) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, j2) self.assertEqual(len(w), 0) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, k2) self.assertEqual(len(w), 0) with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(None, l2) self.assertEqual(len(w), 0)