def parse(self, response):
        response_sel = Selector(response)

        next_link = list_first_item(response_sel.xpath(u'//div[@class="k2"]/div/a[text()="下一页"]/@href').extract())

        if next_link:
            next_link = clean_url(response.url,next_link, response.encoding)
            yield Request(url=next_link,callback=self.parse)

        for detail_link in response_sel.xpath(u'//div[contains(@class,"sousuolist")]/a/@href').extract():
            if detail_link:
                detail_link = clean_url(response.url,detail_link,response.encoding)
                yield Request(url=detail_link, callback=self.parse_detail)
Exemple #2
0
    def parse(self, response):
        response_sel = Selector(response)

        next_link = list_first_item(
            response_sel.xpath(
                u'//div[@class="k2"]/div/a[text()="下一页"]/@href').extract())

        if next_link:
            next_link = clean_url(response.url, next_link, response.encoding)
            yield Request(url=next_link, callback=self.parse)

        for detail_link in response_sel.xpath(
                u'//div[contains(@class,"sousuolist")]/a/@href').extract():
            if detail_link:
                detail_link = clean_url(response.url, detail_link,
                                        response.encoding)
                print detail_link
                yield Request(url=detail_link, callback=self.parse_detail)
    def parse(self, response):
        response_sel = Selector(response)

        next_link = list_first_item(response_sel.xpath(u'//div[@class="page"]/a[text()="下一页"]/@href').extract())

        if next_link:
            next_link = clean_url(response.url, next_link, response.encoding)
            yield Request(url=next_link, callback=self.parse)

        print next_link
        # 必须使用for循环来调用parse_detail函数,否则只能解析第一个界面
        for item in self.parse_detail(response):
            yield item
Exemple #4
0
    def parse(self, response):
        response_sel = Selector(response)

        next_link = list_first_item(
            response_sel.xpath(
                u'//div[@class="page"]/a[text()="下一页"]/@href').extract())

        if next_link:
            next_link = clean_url(response.url, next_link, response.encoding)
            yield Request(url=next_link, callback=self.parse)

        print next_link
        # 必须使用for循环来调用parse_detail函数,否则只能解析第一个界面
        for item in self.parse_detail(response):
            yield item