def parse_detail(self,response):
data_item = DataTreasureItem()
response_selector = Selector(response)
data_item['book_name'] = list_first_item(response_selector.xpath('//div[@class="zizida"][1]/text()').extract())
data_item['book_description'] = list_first_item(response_selector.xpath('//div[@class="lili"][1]/text()').extract()).strip()
yield data_item
def parse_detail(self, response):
response_sel = Selector(response)
table_bodys = response_sel.xpath('//*[@id="dataTables-example"]/tbody/tr')
for table_body in table_bodys:
proxy_item = TutorialItem()
port_url = str(list_first_item(table_body.xpath('./td[2]/img/@src').extract())).split('&')[0]
if port_url in self.proxy_porturl:
proxy_item['proxy_url'] = list_first_item(table_body.xpath('./td[1]/text()').extract()) + ':' + self.proxy_porturl[port_url]
proxy_item['proxy_type'] = list_first_item(table_body.xpath('./td[3]/text()').extract())
proxy_item['proxy_locate'] = list_first_item(table_body.xpath('./td[7]/text()').extract())
else:
continue
yield proxy_item
def parse_detail(self, response):
data_item = DataTreasureItem()
response_selector = Selector(response)
print '********************book name is ' + list_first_item(
response_selector.xpath(
'//div[@class="zizida"][1]/text()').extract())
data_item['book_name'] = list_first_item(
response_selector.xpath(
'//div[@class="zizida"][1]/text()').extract())
data_item['book_description'] = list_first_item(
response_selector.xpath(
'//div[@class="lili"][1]/text()').extract())
yield data_item
def parse(self, response):
response_sel = Selector(response)
next_link = list_first_item(response_sel.xpath(u'//div[@class="k2"]/div/a[text()="下一页"]/@href').extract())
if next_link:
next_link = clean_url(response.url,next_link, response.encoding)
yield Request(url=next_link,callback=self.parse)
for detail_link in response_sel.xpath(u'//div[contains(@class,"sousuolist")]/a/@href').extract():
if detail_link:
detail_link = clean_url(response.url,detail_link,response.encoding)
yield Request(url=detail_link, callback=self.parse_detail)
def parse(self, response):
response_sel = Selector(response)
next_link = list_first_item(response_sel.xpath(u'//div[@class="page"]/a[text()="下一页"]/@href').extract())
if next_link:
next_link = clean_url(response.url, next_link, response.encoding)
yield Request(url=next_link, callback=self.parse)
print next_link
# 必须使用for循环来调用parse_detail函数,否则只能解析第一个界面
for item in self.parse_detail(response):
yield item
def parse_detail(self, response):
response_sel = Selector(response)
table_bodys = response_sel.xpath(
'//*[@id="dataTables-example"]/tbody/tr')
for table_body in table_bodys:
proxy_item = TutorialItem()
port_url = str(
list_first_item(
table_body.xpath('./td[2]/img/@src').extract())).split(
'&')[0]
if port_url in self.proxy_porturl:
proxy_item['proxy_url'] = list_first_item(
table_body.xpath('./td[1]/text()').extract()
) + ':' + self.proxy_porturl[port_url]
proxy_item['proxy_type'] = list_first_item(
table_body.xpath('./td[3]/text()').extract())
proxy_item['proxy_locate'] = list_first_item(
table_body.xpath('./td[7]/text()').extract())
else:
continue
yield proxy_item
def parse(self, response):
response_sel = Selector(response)
next_link = list_first_item(
response_sel.xpath(
u'//div[@class="page"]/a[text()="下一页"]/@href').extract())
if next_link:
next_link = clean_url(response.url, next_link, response.encoding)
yield Request(url=next_link, callback=self.parse)
print next_link
# 必须使用for循环来调用parse_detail函数,否则只能解析第一个界面
for item in self.parse_detail(response):
yield item
def parse(self, response):
response_sel = Selector(response)
next_link = list_first_item(
response_sel.xpath(
u'//div[@class="k2"]/div/a[text()="下一页"]/@href').extract())
if next_link:
next_link = clean_url(response.url, next_link, response.encoding)
yield Request(url=next_link, callback=self.parse)
for detail_link in response_sel.xpath(
u'//div[contains(@class,"sousuolist")]/a/@href').extract():
if detail_link:
detail_link = clean_url(response.url, detail_link,
response.encoding)
print detail_link
yield Request(url=detail_link, callback=self.parse_detail)
