def parse(self, response): response_sel = Selector(response) next_link = list_first_item(response_sel.xpath(u'//div[@class="k2"]/div/a[text()="下一页"]/@href').extract()) if next_link: next_link = clean_url(response.url,next_link, response.encoding) yield Request(url=next_link,callback=self.parse) for detail_link in response_sel.xpath(u'//div[contains(@class,"sousuolist")]/a/@href').extract(): if detail_link: detail_link = clean_url(response.url,detail_link,response.encoding) yield Request(url=detail_link, callback=self.parse_detail)
def parse(self, response): response_sel = Selector(response) next_link = list_first_item( response_sel.xpath( u'//div[@class="k2"]/div/a[text()="下一页"]/@href').extract()) if next_link: next_link = clean_url(response.url, next_link, response.encoding) yield Request(url=next_link, callback=self.parse) for detail_link in response_sel.xpath( u'//div[contains(@class,"sousuolist")]/a/@href').extract(): if detail_link: detail_link = clean_url(response.url, detail_link, response.encoding) print detail_link yield Request(url=detail_link, callback=self.parse_detail)
def parse(self, response): response_sel = Selector(response) next_link = list_first_item(response_sel.xpath(u'//div[@class="page"]/a[text()="下一页"]/@href').extract()) if next_link: next_link = clean_url(response.url, next_link, response.encoding) yield Request(url=next_link, callback=self.parse) print next_link # 必须使用for循环来调用parse_detail函数,否则只能解析第一个界面 for item in self.parse_detail(response): yield item
def parse(self, response): response_sel = Selector(response) next_link = list_first_item( response_sel.xpath( u'//div[@class="page"]/a[text()="下一页"]/@href').extract()) if next_link: next_link = clean_url(response.url, next_link, response.encoding) yield Request(url=next_link, callback=self.parse) print next_link # 必须使用for循环来调用parse_detail函数,否则只能解析第一个界面 for item in self.parse_detail(response): yield item