def test_invalid_subclass():
"""Classes should inherit from ItemPage."""
class MyClass(object):
pass
with pytest.raises(TypeError) as exc:
callback_for(MyClass)
msg = 'MyClass should be a subclass of ItemPage.'
assert str(exc.value) == msg
def test_not_implemented_method():
"""Classes should implement to_item method."""
class MyClass(ItemPage):
pass
with pytest.raises(NotImplementedError) as exc:
callback_for(MyClass)
msg = 'MyClass should implement to_item method.'
assert str(exc.value) == msg
class BooksSpider(scrapy.Spider):
name = 'books_02_1'
start_urls = ['http://books.toscrape.com/']
parse_book = callback_for(BookPage)
def parse(self, response):
for url in response.css('.image_container a::attr(href)').getall():
yield response.follow(url, self.parse_book)
def test_callback_for():
"""Simple test case to ensure it works as expected."""
cb = callback_for(FakeItemPage)
assert callable(cb)
fake_page = FakeItemPage()
response = DummyResponse('http://example.com/')
result = cb(response=response, page=fake_page)
assert list(result) == ['fake item page']
def test_inline_callback():
"""Sample request with inline callback."""
spider = MySpider()
cb = callback_for(FakeItemPage)
request = scrapy.Request('http://example.com/', callback=cb)
with pytest.raises(ValueError) as exc:
request_to_dict(request, spider)
msg = f'Function {cb} is not an instance method in: {spider}'
assert str(exc.value) == msg
class MySpider(scrapy.Spider):
name = 'foo'
custom_settings = {
"SCRAPY_POET_PROVIDERS": {
ResponseDataProvider: 1,
DummyProductProvider: 2,
FakeProductProvider: 3,
}
}
callback_for_parse = callback_for(DummyProductPage)
def parse(self, response):
pass
def parse2(self, res):
pass
def parse3(self, response: DummyResponse):
pass
def parse4(self, res: DummyResponse):
pass
def parse5(self, response, book_page: BookPage):
pass
def parse6(self, response: DummyResponse, book_page: BookPage):
pass
def parse7(self, response, book_page: DummyProductPage):
pass
def parse8(self, response: DummyResponse, book_page: DummyProductPage):
pass
def parse9(self, response, book_page: FakeProductPage):
pass
def parse10(self, response: DummyResponse, book_page: FakeProductPage):
pass
def parse11(self, response: TextResponse):
pass
def parse12(self, response: TextResponse, book_page: DummyProductPage):
pass
def start_requests(self):
yield Request(self.url,
capture_exceptions(callback_for(injectable)))
def parse(self, response, page: BookListPage):
for url in page.book_urls():
yield response.follow(url, callback_for(BookPage))
def parse(self, response):
for url in response.css('.image_container a::attr(href)').getall():
yield response.follow(url, callback_for(ProductPage))
def parse(self, response, page: BookListPage):
yield from response.follow_all(page.book_urls(),
callback_for(BookPage))
def test_callback_for_inline():
callback = callback_for(FakeItemPage)
response = DummyResponse('http://example.com/')
fake_page = FakeItemPage()
result = callback(response, page=fake_page)
assert list(result) == ['fake item page']
class MySpider(scrapy.Spider):
name = 'my_spider'
parse_item = callback_for(FakeItemPage)
parse_web = callback_for(FakeItemWebPage)