def parse_page(self, response): sel = Selector(response) nsil = NewsScraperItemLoader(selector=sel) # data pre-processing author_raw = nsil.get_xpath("//p[contains(@itemprop, 'author')]//text()", NormalizedJoin()) date_raw = nsil.get_xpath("//p[contains(@itemprop, 'author')]//text()", NormalizedJoin()) author = "".join(AUTHOR_RE.findall(author_raw)) date = "".join(DATE_RE.findall(date_raw)) # article data first nsil.add_xpath("headline", "//header/h1[@class='heading']//text()", Join()) nsil.add_value("publication", "Ars Technica") nsil.add_value("date_published", date) nsil.add_value("category", response.meta["category"]) nsil.add_value("author", author) nsil.add_xpath("body", "//div[@itemprop='articleBody']//text()", NormalizedJoin()) # metadata nsil.add_value("date_scraped", str(datetime.datetime.now())) nsil.add_value("scraped_by", self.name) nsil.add_value("scraped_from", response.url) yield nsil.load_item()
def parse(self, response): sel = Selector(response) cats = sel.xpath("//ul[@id='main-menu']//li/a") for cat in cats: nsil = NewsScraperItemLoader(selector=cat) url = nsil.get_xpath(".//@href", Join()) category = nsil.get_xpath(".//text()", Join()) yield Request(url=url, callback=self.parse_articles, meta={'category': category})
def parse(self, response): sel = Selector(response) headlines = sel.xpath("//h2[@itemprop='headline']/a") next_button = sel.xpath("//ul[@class='post-pagination']/li[@class='older']/a") for line in headlines: nsil = NewsScraperItemLoader(selector=line) yield Request(url=nsil.get_xpath(".//@href", Join()), callback=self.parse_article) if next_button: nsil = NewsScraperItemLoader(selector=next_button) yield Request(url=urljoin(response.url, nsil.get_xpath(".//@href", Join())), callback=self.parse, dont_filter=True)
def parse(self, response): # grab all the menu links sel = Selector(response) # IIRC the response has a built-in selector now? # need to try that later. menu_links = sel.xpath("//div[@id='main-menu']/div[@id='sections']//a") for link in menu_links: nsil = NewsScraperItemLoader(selector=link) # so the default input/output processor doesn't # fire when you use get_xpath. anchor = nsil.get_xpath(".//@href", Join()) category_name = nsil.get_xpath(".//text()", NormalizedJoin()) yield Request(url=urljoin(response.url, anchor), callback=self.parse_articles, meta={"category": category_name})
def parse_index(self, response): # news and reviews for now. links = response.xpath("//div[@class='headline-wrap']//div[@class='headline']//h3/parent::a") # I'm hoping that this will fire when I expect it to. # that is to say, at the end of the run, when there are # no links left. if not links: raise CloseSpider("No results remaining.") for link in links: nsil = NewsScraperItemLoader(selector=link) rel = nsil.get_xpath(".//@href", Join()) new_url = urljoin(response.url, rel) yield Request(url=new_url, callback=self.parse_article, dont_filter=True) if 'reviews' in response.url: page_rel = "?page=%s" % self.reviews_page_num new_url = urljoin(response.url, page_rel) self.reviews_page_num += 1 print "Sending request for %s" % new_url yield Request(url=new_url, callback=self.parse_index, dont_filter=True) elif 'news' in response.url: page_rel = "?page=%s" % self.news_page_num new_url = urljoin(response.url, page_rel) self.news_page_num += 1 print "Sending request for %s" % new_url yield Request(url=new_url, callback=self.parse_index, dont_filter=True)
def parse(self, response): # the headlines we need to get need to be obtained via 3 # different xpaths. head1 = response.xpath("//ul[@class='headlines medium']//div//h2[@class='title']//a") head2 = response.xpath("//ul[contains(@class, 'headlines large')]//div//h2[@class='title']//a") head3 = response.xpath("//ul[contains(@class, 'headlines small')]//div//h2[@class='title']//a") # for ease of use, all three lists will be combined into one large one # that we can iterate over. headlines = head1 + head2 + head3 if headlines: for head in headlines: nsil = NewsScraperItemLoader(selector=head) anchor = nsil.get_xpath("./@href", Join()) new_url = urljoin(response.url, anchor) yield Request(url=new_url, callback=self.parse_article) if 'news' in response.url: next_anchor = "/ajax.php?action=frontpage&page=%s&platform=&type=news&topic=" % self.page elif 'reviews' in response.url: next_anchor = "/ajax.php?action=frontpage&page=%s&platform=&type=reviews&topic=" % self.page self.page += 1 next_page = urljoin(response.url, next_anchor) yield Request(url=next_page, callback=self.parse, dont_filter=True)
def parse(self, response): links = response.xpath("//div[@class='post-inner']//h2/a") next_button = None for link in links: nsil = NewsScraperItemLoader(selector=link) new_url = nsil.get_xpath(".//@href", Join()) yield Request(url=new_url, callback=self.parse_article) if next_button: nsil = NewsScraperItemLoader(selector=next_button) new_url = nsil.get_xpath("//div[@class='navigations']//a[contains(text(), 'Older')]", Join()) yield Request(url=new_url, callback=self.parse, dont_filter=True)
def parse(self, response): sel = Selector(response) links = sel.xpath("//a[@class='read-more']") next_button = sel.xpath("//li[@class='next']//a") for link in links: nsil = NewsScraperItemLoader(selector=link) new_url = nsil.get_xpath(".//@href", Join()) yield Request(url=new_url, callback=self.parse_article) if next_button: nsil = NewsScraperItemLoader(selector=next_button) new_url = urljoin(response.url, nsil.get_xpath(".//@href", Join())) yield Request(url=new_url, callback=self.parse, dont_filter=True)
def parse(self, response): links = response.xpath("//div[@class='blog_post']//h1//a") next_page = response.xpath("//div[@class='olderposts']//a") for link in links: nsil = NewsScraperItemLoader(selector=link) new_url = nsil.get_xpath(".//@href", Join()) yield Request(url=new_url, callback=self.parse_article) if next_page: nsil = NewsScraperItemLoader(selector=next_page) new_url = nsil.get_xpath(".//@href", Join()) yield Request(url=new_url, callback=self.parse, dont_filter=True)
def parse_articles(self, response): sel = Selector(response) articles = sel.xpath("//div[@class='stories']//ul/li//a") older_posts = sel.xpath("//a[@class='btn-prev']") for art in articles: nsil = NewsScraperItemLoader(selector=art) url = nsil.get_xpath(".//@href", Join()) yield Request(url=url, callback=self.parse_body, meta={'category': response.meta["category"]}) if older_posts: nsil = NewsScraperItemLoader(selector=older_posts) url = nsil.get_xpath(".//@href", Join()) yield Request(url=url, callback=self.parse, dont_filter=True)
def parse(self, response): articles = response.xpath("//article/a") next_button = response.xpath("//div[@class='load-articles']/button") for art in articles: nsil = NewsScraperItemLoader(selector=art) anchor = nsil.get_xpath(".//@href", Join()) new_url = urljoin(response.url, anchor) yield Request(url=new_url, callback=self.parse_article) if next_button: nsil = NewsScraperItemLoader(selector=next_button) anchor = nsil.get_xpath(".//@data-next", Join()) new_url = urljoin(response.url, anchor) yield Request(url=new_url, callback=self.parse_index, dont_filter=True)
def parse(self, response): stuff = response.xpath("//div[@class='post-info']//h4//a") next_page = response.xpath("//div[@class='paginate_right']//a") for post in stuff: nsil = NewsScraperItemLoader(selector=post) anchor = nsil.get_xpath(".//@href", Join()) author = nsil.get_xpath("./../../div[@id='subtitle']/small/a/text()", Join()) new_url = urljoin(response.url, anchor) yield Request(url=new_url, callback=self.parse_article, meta={'author': author}) if next_page: nsil = NewsScraperItemLoader(selector=next_page) anchor = nsil.get_xpath(".//@href", Join()) new_url = urljoin(response.url, anchor) yield Request(url=new_url, callback=self.parse, dont_filter=True)
def parse(self, response): review_links = response.xpath("//ul[@class='global_list reviews']//li//h2/a") # the :-1 should slice it so only the first element is retrieved. next_button = response.xpath("//div[contains(@class, 'gr_pagination')]//a[@title='Next Page']")[:-1] for link in review_links: nsil = NewsScraperItemLoader(selector=link) anchor = nsil.get_xpath(".//@href", Join()) new_url = urljoin(response.url, anchor) yield Request(url=new_url, callback=self.parse_review, dont_filter=True, meta=response.meta) if next_button: nsil = NewsScraperItemLoader(selector=next_button) anchor = nsil.get_xpath(".//@href", Join()) new_url = urljoin(response.url, anchor) yield Request(url=new_url, callback=self.parse, dont_filter=True, meta=response.meta)
def parse(self, response): nsil = NewsScraperItemLoader(selector=response.selector) link_xpath = ["//nav[@class='nav-links']//li/a[contains(., 'News')]/@href", "//nav[@class='nav-links']//li/a[contains(., 'Reviews')]/@href" ] rels = nsil.get_xpath(link_xpath) for rel in rels: new_url = urljoin(response.url, rel) print new_url yield Request(url=new_url, callback=self.parse_index, dont_filter=True)
def parse_articles(self, response): sel = Selector(response) # get articles, yield links to articles # get next page links, yield links to next pages. article_links = sel.xpath("//h1[@class='heading']/a") for link in article_links: nsil = NewsScraperItemLoader(selector=link) title = nsil.get_xpath("./text()") yield Request(url=nsil.get_xpath("./@href", Join()), callback=self.parse_page, meta=response.meta) # this block of code checks to see if there is an 'older posts' # page. articles are listed newest first on arstechnica. older_posts = sel.xpath("//table//td[contains(@class,'older')]/a") if older_posts: nsil = NewsScraperItemLoader(selector=older_posts) anchor = nsil.get_xpath("./@href", Join()) yield Request(url=urljoin(response.url, anchor), callback=self.parse_articles, meta=response.meta, dont_filter=True)
def parse(self, response): links = response.xpath("//div[@data-post-type='post']//h3/a") if links: for link in links: nsil = NewsScraperItemLoader(selector=link) new_url = nsil.get_xpath(".//@href", Join()) yield Request(url=new_url, callback=self.parse_article) # pagination must be done this way since there is no way to tell # in advance how many pages of news there actually are. self.page_num += 1 raw_url = "http://www.digitaltrends.com/page/%s" % self.page_num yield Request(url=raw_url, callback=self.parse, dont_filter=True)