def test_invalid_subclass(): """Classes should inherit from ItemPage.""" class MyClass(object): pass with pytest.raises(TypeError) as exc: callback_for(MyClass) msg = 'MyClass should be a subclass of ItemPage.' assert str(exc.value) == msg
def test_not_implemented_method(): """Classes should implement to_item method.""" class MyClass(ItemPage): pass with pytest.raises(NotImplementedError) as exc: callback_for(MyClass) msg = 'MyClass should implement to_item method.' assert str(exc.value) == msg
class BooksSpider(scrapy.Spider): name = 'books_02_1' start_urls = ['http://books.toscrape.com/'] parse_book = callback_for(BookPage) def parse(self, response): for url in response.css('.image_container a::attr(href)').getall(): yield response.follow(url, self.parse_book)
def test_callback_for(): """Simple test case to ensure it works as expected.""" cb = callback_for(FakeItemPage) assert callable(cb) fake_page = FakeItemPage() response = DummyResponse('http://example.com/') result = cb(response=response, page=fake_page) assert list(result) == ['fake item page']
def test_inline_callback(): """Sample request with inline callback.""" spider = MySpider() cb = callback_for(FakeItemPage) request = scrapy.Request('http://example.com/', callback=cb) with pytest.raises(ValueError) as exc: request_to_dict(request, spider) msg = f'Function {cb} is not an instance method in: {spider}' assert str(exc.value) == msg
class MySpider(scrapy.Spider): name = 'foo' custom_settings = { "SCRAPY_POET_PROVIDERS": { ResponseDataProvider: 1, DummyProductProvider: 2, FakeProductProvider: 3, } } callback_for_parse = callback_for(DummyProductPage) def parse(self, response): pass def parse2(self, res): pass def parse3(self, response: DummyResponse): pass def parse4(self, res: DummyResponse): pass def parse5(self, response, book_page: BookPage): pass def parse6(self, response: DummyResponse, book_page: BookPage): pass def parse7(self, response, book_page: DummyProductPage): pass def parse8(self, response: DummyResponse, book_page: DummyProductPage): pass def parse9(self, response, book_page: FakeProductPage): pass def parse10(self, response: DummyResponse, book_page: FakeProductPage): pass def parse11(self, response: TextResponse): pass def parse12(self, response: TextResponse, book_page: DummyProductPage): pass
def start_requests(self): yield Request(self.url, capture_exceptions(callback_for(injectable)))
def parse(self, response, page: BookListPage): for url in page.book_urls(): yield response.follow(url, callback_for(BookPage))
def parse(self, response): for url in response.css('.image_container a::attr(href)').getall(): yield response.follow(url, callback_for(ProductPage))
def parse(self, response, page: BookListPage): yield from response.follow_all(page.book_urls(), callback_for(BookPage))
def test_callback_for_inline(): callback = callback_for(FakeItemPage) response = DummyResponse('http://example.com/') fake_page = FakeItemPage() result = callback(response, page=fake_page) assert list(result) == ['fake item page']
class MySpider(scrapy.Spider): name = 'my_spider' parse_item = callback_for(FakeItemPage) parse_web = callback_for(FakeItemWebPage)