def similar_check(cls, nodeA, nodeB):
if nodeA.tag != nodeB.tag:
return False
#compare distinct nodes
dnodesA = ElementHelper.get_children(nodeA)
dnodesB = ElementHelper.get_children(nodeB)
#dA is node_levels_mapping, rA is level_nodes_mapping
dA, dB, rA, rB = {}, {}, {}, {}
for node in dnodesA:
#ignore <a> tag as distinct tag
if node.tag == 'a': continue
dA.setdefault(node.tag, []).append(int(node.get(px)))
rA.setdefault(int(node.get(px)), []).append(node.tag)
for node in dnodesB:
if node.tag == 'a': continue
dB.setdefault(node.tag, []).append(int(node.get(px)))
rB.setdefault(int(node.get(px)), []).append(node.tag)
if abs(len(dA)-len(dB))>1 or abs(len(rA)-len(rB))>1:
return False
#check distinct tag is same？
for tag in dA:
if tag not in ('em', 'b', 'br','i', 'font') and tag not in dB:
return False
sumA = sum([len(StringHelper.unique(rA[A])) for A in rA])
sumB = sum([len(StringHelper.unique(rB[B])) for B in rB])
if abs(sumA-sumB)>1:
return False
return True
def similar_check(cls, nodeA, nodeB):
if nodeA.tag != nodeB.tag:
return False
#compare distinct nodes
dnodesA = ElementHelper.get_children(nodeA)
dnodesB = ElementHelper.get_children(nodeB)
#dA is node_levels_mapping, rA is level_nodes_mapping
dA, dB, rA, rB = {}, {}, {}, {}
for node in dnodesA:
#ignore <a> tag as distinct tag
if node.tag == 'a': continue
dA.setdefault(node.tag, []).append(int(node.get(px)))
rA.setdefault(int(node.get(px)), []).append(node.tag)
for node in dnodesB:
if node.tag == 'a': continue
dB.setdefault(node.tag, []).append(int(node.get(px)))
rB.setdefault(int(node.get(px)), []).append(node.tag)
if abs(len(dA)-len(dB))>1 or abs(len(rA)-len(rB))>1:
return False
#check distinct tag is same？
for tag in dA:
if tag not in ('em', 'b', 'br','i', 'font') and tag not in dB:
return False
sumA = sum([len(StringHelper.unique(rA[A])) for A in rA])
sumB = sum([len(StringHelper.unique(rB[B])) for B in rB])
if abs(sumA-sumB)>1:
return False
return True
def split_words(content):
content = StringHelper.chinese_segment(content)
content = content.lower()
content = content.split()
import string
punctuation = [unicode(p) for p in string.punctuation]
chinese_punctuation = u',?、。“”《》！，：；？＞＜＠＃＄％︿＆＊＋～｜［］｛｝-'
chinese_punctuation = StringHelper.chinese_segment(chinese_punctuation).split()
punctuation.extend(chinese_punctuation)
content=[c for c in content if c not in punctuation]
return content
def split_words(content):
content = StringHelper.chinese_segment(content)
content = content.lower()
content = content.split()
import string
punctuation = [unicode(p) for p in string.punctuation]
chinese_punctuation = u',?、。“”《》！，：；？＞＜＠＃＄％︿＆＊＋～｜［］｛｝-'
chinese_punctuation = StringHelper.chinese_segment(chinese_punctuation).split()
punctuation.extend(chinese_punctuation)
content=[c for c in content if c not in punctuation]
return content
def find_first_sibling_record_node(cls, element, doctree):
'''找到element所在区域的起始节点
1.首选查看element的xpath下表，如果其下表<2，表示element左边没有兄弟节点了，直接返回element
2.如果element的xpath下标大于=2,表示element左边有兄弟节点，那么先找到element的父亲（以便于访问element的兄弟节点，然后index=设置为element的下表-2
查看parentt[index]是否是数据区域，如果是继续项左寻找，否则返回parent[index+1]
'''
parent = element.getparent()
if len(parent)<2:
return element
element_xpath = ElementHelper.get_xpath_by_element(element, doctree)
# print 'xpath: %s' %element_xpath
element_last_index = StringHelper.get_digits(element_xpath.split('/')[-1])
if element_last_index < 2:
return element
index = element_last_index - 2
# print 'parent length:%d' %len(parent)
while index >= 0:
# print index
if parent[index].get(kg_record_mark) == '1':
index -= 1
else:
break
return parent[index+1]
def find_first_sibling_record_node(cls, element, doctree):
'''找到element所在区域的起始节点
1.首选查看element的xpath下表，如果其下表<2，表示element左边没有兄弟节点了，直接返回element
2.如果element的xpath下标大于=2,表示element左边有兄弟节点，那么先找到element的父亲（以便于访问element的兄弟节点，然后index=设置为element的下表-2
查看parentt[index]是否是数据区域，如果是继续项左寻找，否则返回parent[index+1]
'''
parent = element.getparent()
if len(parent)<2:
return element
element_xpath = ElementHelper.get_xpath_by_element(element, doctree)
# print 'xpath: %s' %element_xpath
element_last_index = StringHelper.get_digits(element_xpath.split('/')[-1])
if element_last_index < 2:
return element
index = element_last_index - 2
# print 'parent length:%d' %len(parent)
while index >= 0:
# print index
if parent[index].get(kg_record_mark) == '1':
index -= 1
else:
break
return parent[index+1]
def is_node_or_children_record(cls, element):
children = ElementHelper.get_children(element)
marks = [child.get(kg_record_mark) for child in children]
unique_marks = StringHelper.unique(marks)
if len(unique_marks)==2:
return True
return False
def is_node_or_children_record(cls, element):
children = ElementHelper.get_children(element)
marks = [child.get(kg_record_mark) for child in children]
unique_marks = StringHelper.unique(marks)
if len(unique_marks)==2:
return True
return False
def is_cluster_contain_user_comments(cluster):
""" identify whether element or its children contain comment content, only consider <a> tag
1.each node in cluster, at least has 3 children
2.there is at least one <a> tag has same text
"""
# can not identify
if len(cluster) < 2: return False
text_number_mapping = {}
#at least have three children contain text
for node in cluster:
children = ElementHelper.get_children(node)
link_nodes_contain_text = [n for n in children if is_link_node_with_text(n)]
non_link_nodes_contain_text = [n for n in children if is_none_link_node_with_text(n)]
if len(link_nodes_contain_text)<3: return False
if len(non_link_nodes_contain_text)<2: return False
for n in link_nodes_contain_text:
text = ElementHelper.element_text_content(n)
if text in text_number_mapping:
text_number_mapping[text] += 1
else:
text_number_mapping[text] = 1
#去除标点符号，出数字，空的文本
tmp = copy.deepcopy(text_number_mapping)
for text in tmp:
if len(text)==0 or StringHelper.is_digits(text) :
del text_number_mapping[text]
text_number = text_number_mapping.values()
# FOR TEST
# for text, number in node_text_mapping.items():
# print text,':', number
text_number_counter = collections.Counter(text_number).most_common()
for number, counter in text_number_counter:
if number > 1 and number==len(cluster) and counter>=2: #ToDo 2016/03/08 old:counter>=2 --> new:counter>=1
print 'find comment!'
return True
return False
def is_cluster_contain_user_comments(cluster):
""" identify whether element or its children contain comment content, only consider <a> tag
1.each node in cluster, at least has 3 children
2.there is at least one <a> tag has same text
"""
# can not identify
if len(cluster) < 2: return False
text_number_mapping = {}
#at least have three children contain text
for node in cluster:
children = ElementHelper.get_children(node)
link_nodes_contain_text = [n for n in children if is_link_node_with_text(n)]
non_link_nodes_contain_text = [n for n in children if is_none_link_node_with_text(n)]
if len(link_nodes_contain_text)<3: return False
if len(non_link_nodes_contain_text)<2: return False
for n in link_nodes_contain_text:
text = ElementHelper.element_text_content(n)
if text in text_number_mapping:
text_number_mapping[text] += 1
else:
text_number_mapping[text] = 1
#去除标点符号，出数字，空的文本
tmp = copy.deepcopy(text_number_mapping)
for text in tmp:
if len(text)==0 or StringHelper.is_digits(text) :
del text_number_mapping[text]
text_number = text_number_mapping.values()
# FOR TEST
# for text, number in node_text_mapping.items():
# print text,':', number
text_number_counter = collections.Counter(text_number).most_common()
for number, counter in text_number_counter:
if number > 1 and number==len(cluster) and counter>=2: #ToDo 2016/03/08 old:counter>=2 --> new:counter>=1
print 'find comment!'
return True
return False
def ComputeDensitySum(element, ratio):
density_sum, char_num_sum = 0.0, 0
_from, index, length = 0, 0, 0
content = ElementHelper.element_text_content(element)
if ElementHelper.is_element_has_child(element):
for child in element:
ComputeDensitySum(child, ratio)
for child in element:
density_sum += float(child.attrib.get(kg_text_density))
char_num_sum += long(child.attrib.get(kg_char_num))
#text before tag
child_content = ElementHelper.element_text_content(child)
index = -1
if child_content != '':
index = StringHelper.index_of(content, child_content, _from)
if index > -1:
length = index - _from
if length > 0:
try:
tmp = length * qLn(1.0 * length) / qLn(
qLn(ratio * length + qExp(1.0))) #此处的计算结果都为0
density_sum += tmp
except ZeroDivisionError:
pass
_from = index + len(child_content)
#text after tag
length = len(ElementHelper.element_text_content(element)) - _from
if length > 0:
try:
density_sum += length * qLn(1.0 * length) / qLn(
qLn(ratio * length + qExp(1.0)))
except ZeroDivisionError:
pass
else:
density_sum = float(element.attrib.get(kg_text_density))
d2s_density_sum = str(density_sum)
element.set(kg_density_sum, d2s_density_sum)
def find_first_sibling_record_node(cls, element, doctree):
parent = element.getparent()
if len(parent)<2:
return element
element_xpath = ElementHelper.get_xpath_by_element(element, doctree)
# print 'xpath: %s' %element_xpath
element_last_index = StringHelper.get_digits(element_xpath.split('/')[-1])
if element_last_index < 2:
return element
index = element_last_index - 2
# print 'parent length:%d' %len(parent)
while index >= 0:
# print index
if parent[index].get(kg_record_mark) == '1':
index -= 1
else:
break
return parent[index+1]
def ComputeDensitySum(element, ratio):
density_sum, char_num_sum = 0.0, 0
_from, index, length = 0, 0, 0
content = ElementHelper.element_text_content(element)
if ElementHelper.is_element_has_child(element):
for child in element:
ComputeDensitySum(child, ratio)
for child in element:
density_sum += float(child.attrib.get(kg_text_density))
char_num_sum += long(child.attrib.get(kg_char_num))
#text before tag
child_content = ElementHelper.element_text_content(child)
index = -1
if child_content != '':
index = StringHelper.index_of(content, child_content, _from)
if index > -1:
length = index - _from
if length > 0:
try:
tmp = length * qLn(1.0 * length) / qLn(qLn(ratio * length + qExp(1.0))) #此处的计算结果都为0
density_sum += tmp
except ZeroDivisionError:
pass
_from = index + len(child_content)
#text after tag
length = len(ElementHelper.element_text_content(element)) - _from
if length>0:
try:
density_sum += length * qLn(1.0 * length) / qLn(qLn(ratio * length + qExp(1.0)))
except ZeroDivisionError:
pass
else:
density_sum = float(element.attrib.get(kg_text_density))
d2s_density_sum = str(density_sum)
element.set(kg_density_sum, d2s_density_sum)
def replaceAll(pattern, src, tar):
return StringHelper.replaceAll(pattern, src, tar)
def normalize_word(content):
return StringHelper.normalize_word(content)
def replaceAll(pattern, src, tar):
return StringHelper.replaceAll(pattern, src, tar)
def normalize_word(content):
return StringHelper.normalize_word(content)