def __find_attribute_row_coordinate(self, structuring_data):
attributes_row_index_coordinate = {}
for key in self.attributes_map:
coordinates = ArrayHelper.locate_value(structuring_data, key)
for coordinate in coordinates:
x = tuple(coordinate[0:len(coordinate) - 1])
if x in attributes_row_index_coordinate.keys():
counter = attributes_row_index_coordinate[x]
attributes_row_index_coordinate[x] = counter + 1
else:
attributes_row_index_coordinate[x] = 1
attributes_row_coordinate = []
number_of_target_attributes = len(self.get_target_attributes())
for key in attributes_row_index_coordinate:
if attributes_row_index_coordinate[key] >= min(
number_of_target_attributes, 3):
attributes_row_coordinate.append(list(key))
if len(attributes_row_coordinate) == 0:
print("找不到符合的属性行")
if len(attributes_row_coordinate) != 0:
print("所有潜在属性行如下:")
for a in attributes_row_coordinate:
print(
ArrayHelper.get_value_by_coordinate(
structuring_data, a))
return attributes_row_coordinate
def mapping(self, data):
# 计算所有属性行的位置
attributes_row_index_counter = {}
for key in self.col_mapping_dic:
coordinates = ArrayHelper.locate_value(data, key)
for coordinate in coordinates:
x = tuple(coordinate[0:len(coordinate) - 1])
if x in attributes_row_index_counter.keys():
counter = attributes_row_index_counter[x]
attributes_row_index_counter[x] = counter + 1
else:
attributes_row_index_counter[x] = 1
attributes_row_index = []
number_of_target_attributes = len(self.get_target_attributes())
for key in attributes_row_index_counter:
if attributes_row_index_counter[key] >= min(
number_of_target_attributes, 3):
attributes_row_index.append(key)
print("所有可能的属性行如下:")
for index in attributes_row_index:
print(ArrayHelper.get_value_by_coordinate(data, index))
# 映射
last_data_row_index = -1 # 记录上一个数据区的最后一行下标
for coordinate in attributes_row_index:
attribute_row_index = coordinate[len(coordinate) - 1]
start_row_index = attribute_row_index + 1
attribute_row = ArrayHelper.get_value_by_coordinate(
data, coordinate)
table_coordinate = coordinate[0:len(coordinate) - 1]
table = ArrayHelper.get_value_by_coordinate(
data, table_coordinate)
if table is None:
continue
# 计算下标和属性的对应关系
index_to_attribute = {}
select_attributes = []
for i in range(0, len(attribute_row)):
target_attribute = DataMapper.get_target_attribute(
self.col_mapping_dic, attribute_row[i])
if target_attribute is not None:
index_to_attribute[i] = target_attribute
select_attributes.append(attribute_row[i])
for i in range(start_row_index, len(table)):
row = table[i]
current_coordinate = list(coordinate[0:len(coordinate) -
1])
current_coordinate.append(i)
if len(row) != len(attribute_row) or tuple(
current_coordinate) in attributes_row_index:
last_data_row_index = i - 1
break
result = OrderedDict()
for j in range(0, len(row)):
tmp = row[j]
if isinstance(tmp, list):
tmp = ''.join(tmp)
if j in index_to_attribute.keys():
for attribute in index_to_attribute[j]:
result[attribute] = tmp
# 处理select节点
for node in self.select_node:
select_str = LxmlHelper.get_attribute_of_element(
node, "select")
regex_str = LxmlHelper.get_attribute_of_element(
node, "regex")
attribute_str = node.text
# 以'@'开头表示选择的是表头属性或者标题
value = None
if select_str.startswith('@'):
header_result = re.search('(?<=@h)[0-9]*',
select_str)
# 匹配表头
if header_result is not None:
header_index = int(header_result.group())
if regex_str is not None and len(
regex_str) != 0:
search_result = re.search(
regex_str,
select_attributes[header_index])
if search_result is None:
continue
value = search_result.group()
else:
value = select_attributes[header_index]
title_result = re.search('(?<=@t)[0-9]*',
select_str)
# 匹配标题
if title_result is not None:
title_index = int(title_result.group())
target_title_row_index = attribute_row_index - title_index
if target_title_row_index >= 0 and target_title_row_index > last_data_row_index:
title = None
for x in table[target_title_row_index]:
if x is not None and len(x) != 0:
title = x
break
if title is not None:
if regex_str is not None and len(
regex_str) != 0:
search_result = re.search(
regex_str, title)
if search_result is None:
continue
value = search_result.group()
else:
value = title
else:
# TODO 其他选择方式
pass
result[attribute_str] = value
yield result