def parse_detail(self, response):
# 通过item loader加载item
type_name = response.meta.get("type_name", "")
publish_date = response.meta.get("publish_date", "") # 发布时间
item_loader = kjjysItemLoader(item=kjjysItem(), response=response)
image_url = response.css("#UCAP-CONTENT img::attr(src)").extract()
content = response.css(".Zoom").extract_first("")
title = response.meta.get("title", "")
new_image_url = []
if len(image_url) > 0:
for in_url in image_url:
in_url = parse.urljoin(response.url, in_url)
new_image_url.append(in_url)
else:
item_loader.add_value("front_image_path", '--')
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
if len(new_image_url) > 0:
item_loader.add_value("front_image_url", new_image_url)
# else:
# item_loader.add_value("front_image_url", [""])
item_loader.add_value("source_net", self.start_urls[0])
item_loader.add_value("source_name", '中华人民共和国科学技术部')
item_loader.add_value("type_name", type_name)
item_loader.add_value("title", title)
item_loader.add_value("content", content)
item_loader.add_value("publish_time", publish_date)
item_loader.add_value("crawl_time", datetime.datetime.now())
article_item = item_loader.load_item()
yield article_item
def parse_detail(self, response):
# 通过item loader加载item
type_name = response.meta.get("type_name", "")
publish_date = response.meta.get("publish_date", "") # 发布时间
item_loader = kjjysItemLoader(item=kjjysItem(), response=response)
image_url = response.css("#UCAP-CONTENT img::attr(src)").extract()
title = response.css(".h-news .h-title ::text").extract()
result = ""
content = response.css(".main-aticle").extract_first("")
if title:
result = "".join(title)
result = result.strip()
else:
title = response.css(".article.oneColumn.pub_border h1::text").extract()
if title:
result = "".join(title)
result = result.strip()
content = response.css("#UCAP-CONTENT p").extract()
content = "".join(content)
else:
title = response.css(".bd1 tr:nth-child(3) td:nth-child(2) ::text").extract()
content = response.css("#UCAP-CONTENT").extract_first("")
result = "".join(title)
result = result.strip()
new_image_url = []
if len(image_url) > 0:
for in_url in image_url:
in_url = parse.urljoin(response.url, in_url)
new_image_url.append(in_url)
else:
item_loader.add_value("front_image_path", '--')
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
if len(new_image_url) > 0:
item_loader.add_value("front_image_url", new_image_url)
# else:
# item_loader.add_value("front_image_url", [""])
item_loader.add_value("source_net", self.start_urls[0])
item_loader.add_value("source_name", '中华人民共和国科学技术部')
item_loader.add_value("type_name", type_name)
item_loader.add_value("title", result)
item_loader.add_value("content", content)
item_loader.add_value("publish_time", publish_date)
item_loader.add_value("crawl_time", datetime.datetime.now())
article_item = item_loader.load_item()
yield article_item
def parse_detail(self, response):
# 提取文章的具体字段
# title = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first("")
# create_date = response.xpath("//p[@class='entry-meta-hide-on-mobile']/text()").extract()[0].strip().replace("·","").strip()
# praise_nums = response.xpath("//span[contains(@class, 'vote-post-up')]/h10/text()").extract()[0]
# fav_nums = response.xpath("//span[contains(@class, 'bookmark-btn')]/text()").extract()[0]
# match_re = re.match(".*?(\d+).*", fav_nums)
# if match_re:
# fav_nums = match_re.group(1)
#
# comment_nums = response.xpath("//a[@href='#article-comment']/span/text()").extract()[0]
# match_re = re.match(".*?(\d+).*", comment_nums)
# if match_re:
# comment_nums = match_re.group(1)
#
# content = response.xpath("//div[@class='entry']").extract()[0]
#
# tag_list = response.xpath("//p[@class='entry-meta-hide-on-mobile']/a/text()").extract()
# tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
# tags = ",".join(tag_list)
# 通过css选择器提取字段
# front_image_url = response.meta.get("front_image_url", "") #文章封面图
# title = response.css(".entry-header h1::text").extract()[0]
# create_date = response.css("p.entry-meta-hide-on-mobile::text").extract()[0].strip().replace("·","").strip()
# praise_nums = response.css(".vote-post-up h10::text").extract()[0]
# fav_nums = response.css(".bookmark-btn::text").extract()[0]
# match_re = re.match(".*?(\d+).*", fav_nums)
# if match_re:
# fav_nums = int(match_re.group(1))
# else:
# fav_nums = 0
#
# comment_nums = response.css("a[href='#article-comment'] span::text").extract()[0]
# match_re = re.match(".*?(\d+).*", comment_nums)
# if match_re:
# comment_nums = int(match_re.group(1))
# else:
# comment_nums = 0
#
# content = response.css("div.entry").extract()[0]
#
# tag_list = response.css("p.entry-meta-hide-on-mobile a::text").extract()
# tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
# tags = ",".join(tag_list)
#
# article_item["url_object_id"] = get_md5(response.url)
# article_item["title"] = title
# article_item["url"] = response.url
# try:
# create_date = datetime.datetime.strptime(create_date, "%Y/%m/%d").date()
# except Exception as e:
# create_date = datetime.datetime.now().date()
# article_item["create_date"] = create_date
# article_item["front_image_url"] = [front_image_url]
# article_item["praise_nums"] = praise_nums
# article_item["comment_nums"] = comment_nums
# article_item["fav_nums"] = fav_nums
# article_item["tags"] = tags
# article_item["content"] = content
# 通过item loader加载item
article_item = kjjysItem()
type_name = response.meta.get("type_name", "")
publish_date = response.meta.get("publish_date", "") # 发布时间
item_loader = kjjysItemLoader(item=kjjysItem(), response=response)
image_url = response.css("#xw_box img::attr(src)").extract()
new_image_url = [
'http://wx3.sinaimg.cn/mw690/7cc829d3gy1fsrtjp2o93j20hs0audih.jpg'
]
if len(image_url) > 0:
for in_url in image_url:
in_url = parse.urljoin(response.url, in_url)
new_image_url.append(in_url)
else:
item_loader.add_value("front_image_path", '--')
item_loader.add_value("url", response.url)
item_loader.add_value("url_object_id", get_md5(response.url))
if len(new_image_url) > 0:
item_loader.add_value("front_image_url", new_image_url)
# else:
# item_loader.add_value("front_image_url", [""])
item_loader.add_value("source_net", self.start_urls[0])
item_loader.add_value("source_name", '科技教育司')
item_loader.add_value("type_name", type_name)
item_loader.add_css("title", "div.tit ::text")
item_loader.add_xpath("content", "//*[@id='xw_box']/p")
item_loader.add_value("publish_time", publish_date)
item_loader.add_value("crawl_time", datetime.datetime.now())
article_item = item_loader.load_item()
yield article_item