def call_spider(self, result, request, spider):
result.request = request
dfd = defer_result(result)
callback = request.callback or spider.parse
warn_on_generator_with_return_value(spider, callback)
warn_on_generator_with_return_value(spider, request.errback)
dfd.addCallbacks(callback=callback,
errback=request.errback,
callbackKeywords=request.cb_kwargs)
return dfd.addCallback(iterate_spider_output)
def call_spider(self, result, request, spider):
if isinstance(result, Response):
callback = request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
result.request = request
return iterate_spider_output(callback(result))
elif request.errback is not None:
warn_on_generator_with_return_value(spider, request.errback)
return iterate_spider_output(request.errback(result))
else:
return result
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred:
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
callback = result.request.callback or spider._parse
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
dfd.addCallbacks(callback=callback, callbackKeywords=result.request.cb_kwargs)
else: # result is a Failure
result.request = request
warn_on_generator_with_return_value(spider, request.errback)
dfd = defer_fail(result)
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def call_spider(self, result, request, spider):
if isinstance(result, Response): #从spider中拿到的事Response对象
if getattr(result, "request", None) is None:
result.request = request
callback = result.request.callback or spider._parse # 从request对象里面拿到 对应的callback 否则传入spider的_parse函数作为callback
warn_on_generator_with_return_value(spider, callback)
dfd = defer_succeed(result)
dfd.addCallback(callback, **result.request.cb_kwargs
) # 将spider的callback 添加到 deferred的回调链路上
else: # result is a Failure
result.request = request
warn_on_generator_with_return_value(spider, request.errback)
dfd = defer_fail(result)
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def test_generators_return_something(self):
def f1():
yield 1
return 2
def g1():
yield 1
return "asdf"
def h1():
yield 1
def helper():
return 0
yield helper()
return 2
def i1():
"""
docstring
"""
url = """
https://example.org
"""
yield url
return 1
assert is_generator_with_return_value(top_level_return_something)
assert is_generator_with_return_value(f1)
assert is_generator_with_return_value(g1)
assert is_generator_with_return_value(h1)
assert is_generator_with_return_value(i1)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None,
top_level_return_something)
self.assertEqual(len(w), 1)
self.assertIn(
'The "NoneType.top_level_return_something" method is a generator',
str(w[0].message))
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, f1)
self.assertEqual(len(w), 1)
self.assertIn('The "NoneType.f1" method is a generator',
str(w[0].message))
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, g1)
self.assertEqual(len(w), 1)
self.assertIn('The "NoneType.g1" method is a generator',
str(w[0].message))
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, h1)
self.assertEqual(len(w), 1)
self.assertIn('The "NoneType.h1" method is a generator',
str(w[0].message))
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, i1)
self.assertEqual(len(w), 1)
self.assertIn('The "NoneType.i1" method is a generator',
str(w[0].message))
def test_indentation_error(self):
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, top_level_return_none)
self.assertEqual(len(w), 1)
self.assertIn('Unable to determine', str(w[0].message))
def test_generators_return_none(self):
def f2():
yield 1
return None
def g2():
yield 1
return
def h2():
yield 1
def i2():
yield 1
yield from generator_that_returns_stuff()
def j2():
yield 1
def helper():
return 0
yield helper()
def k2():
"""
docstring
"""
url = """
https://example.org
"""
yield url
return
def l2():
return
assert not is_generator_with_return_value(top_level_return_none)
assert not is_generator_with_return_value(f2)
assert not is_generator_with_return_value(g2)
assert not is_generator_with_return_value(h2)
assert not is_generator_with_return_value(i2)
assert not is_generator_with_return_value(j2) # not recursive
assert not is_generator_with_return_value(k2) # not recursive
assert not is_generator_with_return_value(l2)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, top_level_return_none)
self.assertEqual(len(w), 0)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, f2)
self.assertEqual(len(w), 0)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, g2)
self.assertEqual(len(w), 0)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, h2)
self.assertEqual(len(w), 0)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, i2)
self.assertEqual(len(w), 0)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, j2)
self.assertEqual(len(w), 0)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, k2)
self.assertEqual(len(w), 0)
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, l2)
self.assertEqual(len(w), 0)