def parse(self, response):
selector = Selector(response.text)
for quote in selector.xpath('//div[@class="quote"]'):
text = quote.xpath('.//span[@itemprop="text"]')[0].text
author = quote.xpath('.//small[@itemprop="author"]')[0].text
author_url = quote.xpath('.//span/a/@href')[0].text
author_url = urljoin(str(response.url), author_url)
tags = quote.xpath('.//div[@class="tags"]/a').text
self.log(
'quote: %s',
dict(text=text,
tags=tags,
author=author,
author_url=author_url))
next_page_url = selector.xpath('//li[@class="next"]/a/@href')[0].text
self.log('next page url: %s', next_page_url)
async def parse(self, response):
selector = Selector(response.text)
tags = selector.xpath("//div[contains(@class, 'tags-box')]//a").text
self.log("Top ten tags: %s", tags)
yield HttpRequest("http://quotes.toscrape.com/", callback=self.parse)
</div>
</div>
</body>
</html>
'''
if __name__ == '__main__':
from xpaw import Selector
selector = Selector(text)
print('# CSS Selector, content of quotes:')
for quote in selector.css('div.quote'):
print(quote.css('span.text')[0].text)
print('# XPath, content of quotes:')
for quote in selector.xpath('//div[@class="quote"]'):
print(quote.xpath('.//span[@class="text"]')[0].text)
print('# CSS Selector, content of quotes, with HTML tags:')
for quote in selector.css('div.quote'):
print(quote.css('span.text')[0].string)
print('# CSS Selector, quote tags')
for quote in selector.css('div.quote'):
print(quote.css('a.tag').text)
print('# CSS Selector, author urls')
for quote in selector.css('div.quote'):
print(quote.css('small+a')[0].attr('href'))