def get_txt_page_encode(raw:str) -> str:
try:
encode = StringHelper.extra_equ_value(raw, 'charset', '"')
encode = StringHelper.cutfrom(encode, '/')
return encode
except HttpConnectionFailedException as e:
raise e
except:
return ''
def get_page_encode(self, url):
try:
raw = self.req.get(url)
encode = StringHelper.extra_equ_value(raw, 'charset', '"')
encode = StringHelper.cutfrom(encode,'/')
return encode
except HttpConnectionFailedException as e:
raise e
except:
return ''
def ruler_finder_ex(tag, ruler_pair, need_tag=False):
"""
html 页面元素抓取核心函数
:param tag: 待遍历 的 soup tag 元素
:param ruler_pair: 元素寻找规则
:param need_tag: 如果为 true 则返回 找到的 子 tag 元素
:return:
"""
result = None
ruler2 = ruler_pair[1]
next_sibling = False
# 判断是否需要获取 nextSibling
if ExtraHtml.ruler_finder_condition_next_sibling(ruler_pair[0]):
next_sibling = True
# 通过标签中间的内容判断 <span>你好</span> 则 ruler 为 [你好]
if ExtraHtml.ruler_finder_condition_content(ruler2):
try:
if tag.get_text() == ExtraHtml.ruler_finder_condition_content(ruler2, True):
if next_sibling:
result = ExtraHtml.finder_need_tag(tag, need_tag, next_sibling=True, ruler_pair=ruler_pair)
else:
result = (ruler_pair[0], ExtraHtml.finder_need_tag(tag, need_tag))
return result
except:
pass
#在某个父tag 下取得所有相同类型的子tag
# 这个好像是个废掉的功能
elif ExtraHtml.ruler_finder_condition(ruler2) == 'at_loop':
re = ExtraHtml.ruler_finder_multi_at(tag, ruler2)
if re is not None:
result = (ruler_pair[0], re)
return result
#依据父属性做查找
elif ruler_pair[1].count(':') > 0 and ruler_pair[1].count(':') > ruler_pair[1].count('http:'):
status = False
_pair = ruler_pair[1].split(':', 1)
name = _pair[0]#可能 是子标签
name_part = []
if name.count(' ') > 0:#可能 有属性
name_part = name.split(' ', 1)#可能 属性
name = name_part[0]#可能 标签
if tag.name != name:#可能 不是此标签，跳过
return None
# 可能 父标签部分
parent_pair = _pair[1].split(' ', 1)
# 可能 父标签名
parent_name = parent_pair[0]
# 可能 被遍历的tag没有父元素 或者 父元素 不是 tag 或者 父元素名不对 跳过
if tag.parent == None or not isinstance(tag.parent, Tag) or tag.parent.name != parent_name:
return None
# 可能 没有添加属性 那么 父元素的要求已达到
if len(parent_pair) == 1:
status = True
# 父标签的属性
parent_attr = parent_pair[1].split('=', 1)
# 可能* 如果有父标签的属性 或者 等号后为空
if len(parent_attr) == 1 or (len(parent_attr) > 1 and parent_attr[1] == None or parent_attr[1] == ''):
try:
attr = tag.parent.attrs[parent_attr[0]]
status = True
except:
pass
if len(parent_attr) > 1 and parent_attr[1] != None or parent_attr[1] != '' :
try:
if ' '.join(tag.parent.attrs[parent_attr[0]]) == parent_attr[1]:
status = True
except:
pass
if status:
if len(name_part) > 1 and name_part[1] != None and name_part != '':
return ExtraHtml.ruler_finder_ex(tag,(ruler_pair[0], _pair[0]), need_tag)
else:
# result = (ruler_pair[0], tag.text)
result = (ruler_pair[0], ExtraHtml.finder_need_tag(tag, need_tag))
#依据自身属性
elif ruler_pair[1].count(' ') > 0 or ruler_pair[1].count('=') == 2:
if ruler_pair[1].count('=') == 2:
ruler_pair_part_of = ruler_pair[1].split(' ', 2)
identify = ruler_pair_part_of[2].split('=', 1)
if tag.name != ruler_pair_part_of[0]:
return None
try:
if tag.attrs[identify[0]] == identify[1]:
return ExtraHtml.ruler_finder_ex(tag, (ruler_pair[0], ruler_pair_part_of[0] + ' ' + ruler_pair_part_of[1]), need_tag)
except:
pass
else:
_pair = ruler_pair[1].split(' ', 1)
_name = _pair[0]
next_flag = False
if _name.count('^') == 1:
next_flag = True
_name = Sh.cut_head(_name, '^')
if tag.name != _name:
return None
# 关键字查询
keyword = Sh.extra_a_to_b_x(_pair[1], '[', ']')
if _pair[1].count('[') == 1 and _pair[1].count(']') == 1:
if keyword in tag.text and len(tag.contents) <= 1:
if next_flag:
# return (ruler_pair[0], tag.next_sibling.text)
return (ruler_pair[0], ExtraHtml.finder_need_tag(tag, need_tag, True))
else:
# return (ruler_pair[0], tag.text)
return (ruler_pair[0], ExtraHtml.finder_need_tag(tag, need_tag))
equ_pair = _pair[1].split('=',1)
_attr = equ_pair[0]
val = StringHelper.extra_a_to_b(_attr, '(', ')')
_attr = StringHelper.delete_piece(_attr, val)
for item in equ_pair:
if item == '':
equ_pair.remove(item)
if len(equ_pair) > 1:
value = equ_pair[1]
try:
att = tag.attrs[_attr]
_value = ''
if isinstance(att, str):
_value = att
elif isinstance(att, list):
_value = " ".join(tag.attrs[_attr])
if _value == value:
# result = (ruler_pair[0], tag.text)
result = (ruler_pair[0], ExtraHtml.finder_need_tag(tag, need_tag))
except:
pass
else:
try:
val = val[1:len(val) - 1]
ed = tag.attrs[_attr]
op = val
# com = op + ed
com = RulerExtra.canwecom(op, ed)
result = (ruler_pair[0], com)
except:
return None
# return (ruler_pair[0], tag.attrs[_attr])
# 不通过属性 直接查找
else:
if tag.name == ruler_pair[1]:
if tag.text is not None and tag.text != '':
# result = (ruler_pair[0], tag.text)
result = (ruler_pair[0], ExtraHtml.finder_need_tag(tag, need_tag))
# 星判断
if result != None and len(result) > 1 and result[0].count('*') > 0:
key = ''
value = ''
value_list = []
if ruler_pair[0].count('*') > 0:
if len(tag.contents) > 1:#多重判断
for con in tag:
if isinstance(con, Tag):
#提取p标签
if ruler_pair[0].count('***') == 1:
if con.name == 'p':
# value += con.get_text()
# value += ExtraHtml.ruler_finder_recursion_dig_for_p(con)
ExtraHtml.ruler_finder_recursion_dig_for_p(con, value_list)
# 删除所有标签
elif ruler_pair[0].count('**') == 1:
value += con.get_text()
# 保留标签
elif ruler_pair[0].count('*') == 1:
value += str(con)
else:#单tag 判断
if ruler_pair[0].count('**') == 1:
value += tag.get_text()
elif ruler_pair[0].count('*') == 1:
value += str(tag)
#善后
if ruler_pair[0].count('***') == 1:
# value = "".join(value.split())
value = "".join(value_list)
value = "".join(value.split())
key = StringHelper.cutfrom(ruler_pair[0], '***')
elif ruler_pair[0].count('**') == 1:
value = "".join(value.split())
key = StringHelper.cutfrom(ruler_pair[0], '**')
elif ruler_pair[0].count('*') == 1:
key = StringHelper.cutfrom(ruler_pair[0], '*')
elif ruler_pair[0].count('^') > 0:
value = tag.text
value = "".join(value.split())
key = StringHelper.cutfrom(ruler_pair[0], '^')
else:
key = ruler_pair[0]
value = tag.text
if value == '':
print('info: in ruler_finder_ex star filter -> ' + result[0] + ' got empty result!')
value = result[1]
return (key, value)
elif result == None:
return None
else:
return result