def parse_itme(self, response):
article_p = response.xpath('//div[@class="content"]//p')
title_index = []
for i, p in enumerate(article_p):
if p.xpath("./strong").extract():
title_index.append(i)
for s_index, i_item in enumerate(title_index):
category = response.xpath("//h1/text()").extract()[0]
title = article_p[i_item].xpath("./strong/text()").extract()[0]
try:
next_tile = title_index[s_index + 1]
content_str = [
"".join(item.xpath("./text()").extract())
for item in article_p[i_item:next_tile]
]
except Exception as e:
content_str = [
"".join(item.xpath("./text()").extract())
for item in article_p[i_item:]
]
item = TangspiderframeItem()
item['url'] = response.url
item['category'] = category
item['content'] = "".join([item.strip() for item in content_str])
item['title'] = title
yield item
def parse(self, response):
category = response.xpath('//div[@id="place"]/a[2]/text()').extract()[0]
title = response.xpath('//div[@id="info"]/dl/h1/a/text()').extract()[0]
content = response.xpath('//dl[@id="zi"]//p//text()').extract()
item = TangspiderframeItem()
item['url'] = response.url
item['category'] = category
item['content'] = "".join([item.strip() for item in content])
item['title'] = title
yield item
def parse(self, response):
category = response.xpath(
'//div[@class="position"]/a[2]/text()').extract()[0]
title = response.xpath(
'//div[@class="article"]/h1/text()').extract()[0]
content = response.xpath('//div[@class="article"]/p//text()').extract()
item = TangspiderframeItem()
item['url'] = response.url
item['category'] = category
item['content'] = "".join([item.strip() for item in content])
item['title'] = title
yield item
def parse(self, response):
category = response.xpath('//div[@class="dingtou"]/div[@class="da-bre"]/a[2]/text()').extract()
title = response.xpath("//h1/text()").extract() # 文章标题
contents, para = [], []
contents.extend(response.xpath('//div[contains(@class, "article")]//p'))
contents.extend(response.xpath('//div[contains(@class, "zhengwen")]//p'))
contents.extend(response.xpath('//div[contains(@class, "c_body")]//p'))
contents.extend(response.xpath('//div[contains(@class, "cnt_bd")]//p'))
for item in contents:
para.extend(item.xpath("./text()").extract())
sub_tag = item.xpath(".//*")
for sub in sub_tag:
if sub.root.tag != "script":
para.extend(sub.xpath(".//text()").extract())
item = TangspiderframeItem()
item['url'] = response.url
item['category'] = "".join(category).replace(" > ", '').strip()
item['content'] = "".join([item.strip() for item in para])
item['title'] = "".join(title).strip()
yield item