def extract_content(self, param, clean=None, before=None, after=None):
params = {"params": {"selector": "div#eData > dl"}, "method": "select"}
img_params = {
"params": {
"selector": "dd:nth-of-type(1)"
},
"method": "select"
}
text_params = {
"params": {
"selector": "dd:nth-of-type(5)"
},
"method": "select"
}
dls = find_tags(self._soup, params)
content = list()
for dl in dls:
image_tag = find_tag(dl, img_params)
if not image_tag:
continue
url = extract_tag_attribute(image_tag, "text")
if not (url and url.startswith("http") and url.endswith(".jpg")):
continue
content.append({"tag": "img", "src": url})
text_tag = find_tag(dl, text_params)
if not text_tag:
continue
text = extract_tag_attribute(text_tag, "text")
if not text:
continue
text = text.replace("<br />", " ").strip()
content.append({"tag": "p", "text": text})
return self.clean_content(content)
def get_ol_tag(tag):
type_ = extract_tag_attribute(tag, "type")
reverse = extract_tag_attribute(tag, "reversed")
attributes = dict()
if type_:
attributes["type"] = type_
if reverse:
attributes["reversed"] = reverse
return attributes
def get_video_tag(self, tag):
src = extract_tag_attribute(tag, "src")
controls = extract_tag_attribute(tag, "controls")
poster = extract_tag_attribute(tag, "poster")
attributes = dict()
if src:
attributes["src"] = src
if controls:
attributes["controls"] = controls
if poster:
attributes["poster"] = poster
return attributes
def extract_content(self, param, clean=None, before=None, after=None):
content = list()
first_text = {
"params": {
"selector": "p.explain"
},
"method": "select",
"attribute": "text"
}
text = self.find_tag_extract_attribute(self._soup, first_text)
if text:
content.append({"tag": "p", "text": text})
image_list = {
"params": {
"selector": "div.roll > div > ul > li img"
},
"method": "select",
}
for a in find_tags(self._soup, image_list):
src = extract_tag_attribute(a, "src")
if src:
content.append({"tag": "img", "src": src})
for item in content:
if item["tag"] == "img":
item["src"] = item["src"].replace("stn", "n")
return content
def clean_author(self, author):
param = {
"method": "select",
"params": {
"selector": "div#news_template_03_AuthorAndTime > span"
}
}
tag = find_tag(root=self.soup, param=param)
text = extract_tag_attribute(root=tag)
return author.replace(text, "")
def get_gallery(self):
param = {
"method": "select",
"params": {
"selector": "div#tt-slide div.img-wrap"
},
}
tags = find_tags(self.soup, param=param)
images = list()
for tag in tags:
src = extract_tag_attribute(tag, name="data-src")
images.append(src)
return images
def _extract_tag(root, param):
"""
param = {
"method": "find_all",
"params": {},
"nth": 0,
"attribute": "text",
}
:param root:
:type root:
:param param:
:type param:
:return:
:rtype:
"""
tag = find_tag(root, param)
attribute = param.get("attribute")
if attribute is None:
string = extract_tag_attribute(tag, "text")
else:
string = extract_tag_attribute(tag, attribute)
return string
def extract_content(self, param, clean=None, before=None, after=None):
content = list()
top_pic = {
"params": {
"selector": "div.region-inner > figure > img"
},
"method": "select",
"attribute": "src"
}
pic_url = self.find_tag_extract_attribute(self._soup, top_pic)
if pic_url:
pic_url = urljoin(self.url, pic_url)
content.append({"tag": "img", "src": pic_url})
body_items = {
"params": {
"selector": "div.body-copy > div"
},
"method": "select",
}
img_src = {
"params": {
"selector": "img"
},
"method": "select",
"attribute": "src"
}
for a in find_tags(self._soup, body_items):
src = self.find_tag_extract_attribute(a, img_src)
if src:
src = urljoin(self.url, src)
content.append({"tag": "img", "src": src})
else:
text = extract_tag_attribute(a, "text")
if text.replace(" ", "").strip() != "":
content.append({"tag": "p", "text": text})
return content
def find_tag_extract_attribute(root, params):
tag = find_tag(root, params)
if not tag:
return None
return extract_tag_attribute(tag, params.get("attribute", "text"))
def get_object_tag(tag):
data = extract_tag_attribute(tag, "data")
attributes = dict()
if data:
attributes["data"] = data
return attributes