def parse(self, response):
email = response.css('.user span::text').extract_first()
# Add to redis
if email:
redis_connection.sadd('email', email)
for href in response.css("a::attr('href')"):
url = response.urljoin(href.extract())
yield scrapy.Request(url)
def parse_item(self, response):
email = response.css('.user span::text').extract_first()
# Add to redis
redis_connection.sadd(self.name, email)
# Return to process
item = EmailCrawlerItem()
item['email'] = email
return item
