def add_file_to_detail_list(self, match_name):
"""
设定国土行业数据国情的附属文件的验证规则（镶嵌影像）
完成 负责人 王学谦 在这里检验国情的附属文件
:return:
"""
file_main_name = self._object_name
file_path = self.file_info.file_path
# 正则匹配附属文件
if not CUtils.equal_ignore_case(file_path, ''):
match_str = '{0}*.*'.format(match_name)
match_file_list = CFile.file_or_dir_fullname_of_path(
file_path, False, match_str, CFile.MatchType_Common)
match_str_main_name = r'(?i)^{0}[FMP]$'.format(match_name) # 主附属
ext_list = [
'rar', 'zip', 'doc', 'docx', 'xls', 'xlsx', 'txt', 'xml'
]
for file_with_path in match_file_list:
if CUtils.equal_ignore_case(
CFile.file_main_name(file_with_path),
file_main_name): # 去除自身与同名文件
pass
elif CUtils.text_match_re(CFile.file_main_name(file_with_path),
match_str_main_name):
self.add_file_to_details(file_with_path) # 将文件加入到附属文件列表中
elif CFile.file_ext(file_with_path).lower() in ext_list:
self.add_file_to_details(file_with_path)
else:
pass
def get_vector_layercount_and_layers(self, datasource) -> (int, []):
"""
获取矢量数据的图层个数和名称集合
@param datasource:
@return:
"""
layer_count_real = 0
layer_list = []
driver = datasource.GetDriver()
if driver is None:
return layer_count_real, layer_list
# gdb数据层里有一些是内置的拓扑检查的图层, 不要列入这部分为好
shp_ds = datasource
iLayerCount = shp_ds.GetLayerCount()
# print("iLayerCount:" + str(iLayerCount))
for i in range(iLayerCount):
layer = shp_ds.GetLayer(i)
layer_name = layer.GetName()
# print('第{0}个图层：{1}'.format(i, layer_name))
if driver.name == 'OpenFileGDB':
if layer_name.startswith('T_1_'):
continue
elif CUtils.equal_ignore_case(driver.name, 'PGeo'):
if layer_name.upper().endswith('_SHAPE_INDEX'):
continue
elif CUtils.equal_ignore_case(layer_name, 'Selections'):
continue
elif CUtils.equal_ignore_case(layer_name, 'SelectedObjects'):
continue
layer_count_real = layer_count_real + 1
layer_list.append(layer)
return layer_count_real, layer_list
def set_value_from_file(self, file_name: str, file_format: str,
file_encoding: str):
"""
设置value为文件内容
. 根据文件格式, 可以加载文件内容
. 如果文件格式为二进制, 则value存储文件名
:param file_name:
:param file_format:
:param file_encoding:
:return:
"""
if CUtils.equal_ignore_case(file_format, CResource.FileFormat_TXT):
self._value = {
CResource.Name_Text: CFile.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_XML):
self._value = {
CResource.Name_Text: CXml.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_Json):
self._value = {
CResource.Name_Text: CJson.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
else:
self._value = {
CResource.Name_Text: file_name,
CResource.Name_Type: CResource.DataValueType_File
}
def check_src_ib_files_not_locked(self, root_path, parent_path):
"""
检测指定目录下的文件是否没有被锁定
1. 只有所有文件都没有被锁定, 则返回True
1. 如果有任何一个文件被锁定, 则返回False, 而且把文件信息写入message中返回
todo(注意) 这里检查所有文件是否被锁定, 在处理切片数据时, 效率会极慢!!!
:param root_path: 根目录
:param parent_path: 父目录, 在加入提示信息中时, 需要将父目录加入到反馈信息中
:return:
1. 目录下是否全部文件都没有锁定, 都可以入库
1. 被锁定文件的名称列表
"""
parent_path = CFile.join_file(root_path, parent_path)
locked_file_list = CFile.find_locked_file_in_path(parent_path)
more_locked_file = False
max_locked_file_count = len(locked_file_list)
if max_locked_file_count > 3:
max_locked_file_count = 3
more_locked_file = True
message = ''
for locked_file in locked_file_list:
message = CUtils.str_append(
message, CFile.join_file(parent_path, locked_file))
if more_locked_file:
message = CUtils.str_append(message, '...')
if max_locked_file_count > 0:
message = CUtils.str_append(message,
'被其他应用占用了, 无法入库, 请检查解除锁定后重试入库! ')
return max_locked_file_count == 0, message
def classified(self):
"""
设计国土行业数据ortho的验证规则（单景正射）
完成 负责人 王学谦 在这里检验ortho的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext # 初始化需要的参数
file_object_name = file_main_name[:]
file_name_with_full_path = self.file_info.file_name_with_full_path
if file_name_with_full_path.endswith('_21at.xml'):
file_object_name = file_main_name[:-5]
file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name)
check_file_main_name_exist = \
CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) or \
CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Img))
if not check_file_main_name_exist: # 检查主文件存在性
return self.Object_Confirm_IUnKnown, self._object_name
# 检查后缀名
if CUtils.equal_ignore_case(file_ext, self.Name_Tif) or CUtils.equal_ignore_case(file_ext, self.Name_Img):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
file_detail_xml = '{0}_21at.xml'.format(self.file_info.file_main_name_with_full_path)
self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def metadata_bus_dict_process_custom(self, metadata_bus_dict):
"""
对部分需要进行运算的数据进行处理
"""
super().metadata_bus_dict_process_custom(metadata_bus_dict)
centerlatitude = CUtils.dict_value_by_name(metadata_bus_dict, 'centerlatitude', None)
centerlongitude = CUtils.dict_value_by_name(metadata_bus_dict, 'centerlongitude', None)
if (not CUtils.equal_ignore_case(centerlatitude, '')) and (not CUtils.equal_ignore_case(centerlongitude, '')):
try:
try:
db_id = self.file_info.db_server_id
except:
db_id = self.DB_Server_ID_Distribution
if CUtils.equal_ignore_case(db_id, ''):
db_id = self.DB_Server_ID_Distribution
db = CFactory().give_me_db(db_id)
wkt = db.one_row(
'''
select st_astext(st_envelope(st_geomfromewkt(st_astext(st_buffer(st_geographyfromtext(
'POINT({0} {1})'), 5000))))) as wkt
'''.format(centerlatitude, centerlongitude)).value_by_name(0, 'wkt', None)
wkt = wkt.replace('POLYGON((', '').replace('))', '').strip()
coordinates_list = re.split(r'[,]|\s+', wkt)
metadata_bus_dict['bottomleftlatitude'] = coordinates_list[0]
metadata_bus_dict['bottomleftlongitude'] = coordinates_list[1]
metadata_bus_dict['topleftlatitude'] = coordinates_list[2]
metadata_bus_dict['topleftlongitude'] = coordinates_list[3]
metadata_bus_dict['toprightlatitude'] = coordinates_list[4]
metadata_bus_dict['toprightlongitude'] = coordinates_list[5]
metadata_bus_dict['bottomrightlatitude'] = coordinates_list[6]
metadata_bus_dict['bottomrightlongitude'] = coordinates_list[7]
except Exception as error:
raise Exception('通过中心点坐标计算四至坐标时出现异常，详细原因为:{0}'.format(error.__str__()))
def txt_to_xml(self, file_metadata_name_with_path: str):
"""
完成 王学谦 txt文件转xml,在函数外提前定义xml对象并获取父节点传入，函数会将通过父节点构造xml对象 by王学谦
:param file_metadata_name_with_path:查询的mat文件全名，带路径
:return xml_obj:将文件内容存储好的项目对象
"""
text_list = CFile.file_2_list(
file_metadata_name_with_path) # 获取mat文件作为列表
if (text_list is None) or len(text_list) == 0:
raise Exception('元数据文件无法读取，请检查') # 如果获取的文件内容为空，则抛出异常
xml_obj = CXml() # 建立xml对象
node_root = xml_obj.new_xml('root')
xml_obj.set_attr(node_root, self.Name_Type,
self.Transformer_TXT) # 设置root节点与属性
for row_text in text_list:
if not CUtils.equal_ignore_case(row_text, ''):
row_list = re.split(r'=',
row_text.strip()) # 利用正则表达式，根据一个或多个tab剪切字符
# item节点
node_item = xml_obj.create_element(node_root, 'item')
xml_obj.set_attr(node_item, self.Name_Name,
CUtils.any_2_str(row_list[0]).strip())
if len(row_list) > 1:
item_value = CUtils.any_2_str(row_list[1].strip())
if item_value.startswith('"') and item_value.endswith('"'):
item_value = item_value[1:-1]
xml_obj.set_element_text(node_item,
item_value) # 设置item节点与属性与内容
return xml_obj
def classified(self):
self._object_confirm = self.Object_Confirm_IUnKnown
self._object_name = None
current_path = self.file_info.file_name_with_full_path
metadata_file_name = CFile.join_file(current_path,
self.FileName_MetaData_Bus_21AT)
if CFile.file_or_path_exist(metadata_file_name):
self.__bus_metadata_xml_file_name__ = metadata_file_name
self.__metadata_xml_obj__ = CXml()
try:
self.__metadata_xml_obj__.load_file(metadata_file_name)
self.__classified_object_type = CXml.get_element_text(
self.__metadata_xml_obj__.xpath_one(
self.Path_21AT_MD_Content_ProductType))
if CUtils.equal_ignore_case(
self.__classified_object_type,
CUtils.dict_value_by_name(self.get_information(),
self.Plugins_Info_Type,
None)):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = CXml.get_element_text(
self.__metadata_xml_obj__.xpath_one(
self.Path_21AT_MD_Content_ProductName))
except:
self.__metadata_xml_obj__ = None
CLogger().warning('发现文件{0}符合二十一世纪业务数据集标准, 但该文件格式有误, 无法打开! ')
return self._object_confirm, self._object_name
def get_sync_dict_list(self, insert_or_updata) -> list:
"""
insert_or_updata 指明配置的是更新还是插入，-1时为插入，0为更新
本方法的写法为强规则，调用add_value_to_sync_dict_list配置
第一个参数为list，第二个参数为字段名，第三个参数为字段值，第四个参数为特殊配置
"""
sync_dict_list = self.get_sync_predefined_dict_list(insert_or_updata)
object_table_data = self._dataset
# 时间信息
dso_time = object_table_data.value_by_name(0, 'dso_time', '')
dso_time_json = CJson()
dso_time_json.load_obj(dso_time)
self.add_value_to_sync_dict_list(
sync_dict_list, 'begdate',
dso_time_json.xpath_one('start_time', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'enddate', dso_time_json.xpath_one('end_time', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'imagedate',
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', '')))
self.add_value_to_sync_dict_list(
sync_dict_list, 'producetime',
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', '')))
self.add_value_to_sync_dict_list(
sync_dict_list, 'imagedatetag',
self.transform_time_to_imagedatetag(
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', ''))))
if insert_or_updata:
self.add_value_to_sync_dict_list(sync_dict_list, 'isdel', '1')
return sync_dict_list
def __load_table_info(self):
"""
在这里加载数据表的信息, 并初始化__column_list对象
:return:
"""
dict_table_info = self.__database.table_info(self.__table_name)
table_column_list = CUtils.dict_value_by_name(dict_table_info,
self.Name_Columns, None)
self.__column_list.clear()
if table_column_list is None:
return
for table_column in table_column_list:
column_name = CUtils.dict_value_by_name(table_column,
self.Name_Name, '')
if CUtils.equal_ignore_case(column_name, ''):
continue
column_data_type = CUtils.dict_value_by_name(
table_column, self.Name_DataType, '')
column_is_primary_key = CUtils.dict_value_by_name(
table_column, self.Name_PrimaryKey, 0)
self.__column_list.add(
CColumn(column_name, column_data_type,
column_is_primary_key == self.DB_True))
def object_uni_full_name_by_id(self, object_id: str):
result = CFactory().give_me_db(self.db_server_id).one_value(
'''
select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name
from dm2_storage_object, dm2_storage_file, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id
and dm2_storage_file.dsfstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{0}'
and dm2_storage_object.dsoid = :object_id
union
select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name
from dm2_storage_object, dm2_storage_directory, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
and dm2_storage_directory.dsdstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{1}'
and dm2_storage_object.dsoid = :object_id
'''.format(self.FileType_File, self.FileType_Dir)
, {'object_id': object_id}
)
if result is not None:
return result
ds_object_info = CFactory().give_me_db(self.db_server_id).one_row(
'''
select dsoobjectname, dsoparentobjid
from dm2_storage_object
where dsoid = :object_id and dsodatatype not in ('file', 'dir')
'''
)
if ds_object_info.is_empty():
return None
object_name = ds_object_info.value_by_index(0, 0, '')
object_parent_id = ds_object_info.value_by_index(0, 1, '')
if CUtils.equal_ignore_case(object_parent_id, ''):
return None
result = CFactory().give_me_db(self.db_server_id).one_value(
'''
select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name
from dm2_storage_object, dm2_storage_file, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id
and dm2_storage_file.dsfstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{0}'
and dm2_storage_object.dsoid = :object_id
union
select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name
from dm2_storage_object, dm2_storage_directory, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
and dm2_storage_directory.dsdstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{1}'
and dm2_storage_object.dsoid = :object_id
'''.format(self.FileType_File, self.FileType_Dir)
, {'object_id': object_parent_id}
)
if result is not None:
return CFile.join_file(CUtils.any_2_str(result), object_name)
else:
return None
def parser_metadata_spatial_after_qa(self, parser: CMetaDataParser):
"""
在这里直接指定坐标系
"""
result = super().parser_metadata_spatial_after_qa(parser)
try:
Prj_Project = CUtils.dict_value_by_name(self.get_information(), self.Plugins_Info_Coordinate_System, '')
if not CUtils.equal_ignore_case(Prj_Project, ''):
parser.metadata.set_metadata_spatial(
self.DB_True,
'元数据文件[{0}]成功加载! '.format(self.file_info.file_name_with_full_path),
self.Spatial_MetaData_Type_Prj_Project,
Prj_Project
)
parser.metadata.set_metadata_spatial(
self.DB_True,
'元数据文件[{0}]成功加载! '.format(self.file_info.file_name_with_full_path),
self.Spatial_MetaData_Type_Prj_Source,
self.Prj_Source_Custom
)
except Exception as error:
parser.metadata.set_metadata_spatial(
self.DB_False,
'元数据文件[{0}]格式不合法, 无法处理! 详细错误为: {1}'.format(self.file_info.file_name_with_full_path,
error.__str__()),
self.MetaDataFormat_Text,
'')
return CResult.merge_result(self.Exception,
'元数据文件[{0}]格式不合法, 无法处理! '.format(
self.file_info.file_name_with_full_path))
return result
def set_custom_affiliated_file(self):
custom_affiliated_file_list = self.get_custom_affiliated_file_character(
)
if len(custom_affiliated_file_list) > 0:
for affiliated_file_info in custom_affiliated_file_list:
affiliated_file_path = CUtils.dict_value_by_name(
affiliated_file_info, self.Name_FilePath, None)
regex_match = CUtils.dict_value_by_name(
affiliated_file_info, self.Name_RegularExpression, None)
no_match = CUtils.dict_value_by_name(
affiliated_file_info, self.Name_No_Match_RegularExpression,
None)
if (affiliated_file_path is not None) and (regex_match
is not None):
affiliated_file_name_list = CFile.file_or_dir_fullname_of_path(
affiliated_file_path, False, regex_match,
CFile.MatchType_Regex) # 模糊匹配文件列表
if len(affiliated_file_name_list) > 0:
for affiliated_file_name in affiliated_file_name_list:
if no_match is None:
self._object_detail_file_full_name_list.append(
affiliated_file_name)
else:
if not CUtils.text_match_re(
CFile.file_name(affiliated_file_name),
no_match):
self._object_detail_file_full_name_list.append(
affiliated_file_name)
def mat_to_xml(self, file_metadata_name_with_path: str):
"""
完成 王学谦 mat文件转xml,在函数外提前定义xml对象并获取父节点传入，函数会将通过父节点构造xml对象 by王学谦
:param file_metadata_name_with_path:查询的mat文件全名，带路径
:return xml_obj:将文件内容存储好的项目对象
"""
text_list = CFile.file_2_list(
file_metadata_name_with_path) # 获取mat文件作为列表
if (text_list is
None) or len(text_list) == 0 or CUtils.equal_ignore_case(
CUtils.any_2_str(text_list), ''):
raise # 如果获取的文件内容为空，则抛出异常
flag = False # 设置标志
xml_obj = CXml() # 建立xml对象
node_root = xml_obj.new_xml('root')
xml_obj.set_attr(node_root, self.Name_Type,
self.transformer_type) # 设置root节点与属性
for index, row_text in enumerate(text_list):
if row_text.startswith('1\t'): # 从开头为1+tab键的行开始录入
flag = True
row_list = re.split(r'\s+', row_text) # 利用正则表达式，根据一个或多个tab剪切字符
if flag:
node_item = xml_obj.create_element(node_root, 'item')
xml_obj.set_attr(node_item, self.Name_Name,
CUtils.any_2_str(row_list[1]).lower())
xml_obj.set_element_text(
node_item,
CUtils.any_2_str(row_list[2].strip())) # 设置item节点与属性与内容
if not flag:
raise Exception('文件内容异常，无法正常识别文件开头') # 如果未找到1+tab键开头，则抛出异常
return xml_obj
def classified(self):
"""
设计国土行业数据的dem_part_2验证规则
完成 负责人 李宪 在这里检验dem_part_2的识别规则
:return:
"""
super().classified()
file_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name)
check_file_main_name_exist_tif = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
check_file_main_name_exist_bil = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Bil))
if (not check_file_main_name_exist_tif) and (not check_file_main_name_exist_bil):
return self.Object_Confirm_IUnKnown, self._object_name
# 判断是否有‘-’，并且为一个
if not self.file_info.file_main_name.count('-') == 1:
return self.Object_Confirm_IUnKnown, self._object_name
char_1 = self.file_info.file_main_name.split('-')[0]
char_2 = self.file_info.file_main_name.split('-')[1]
# char_1,char_2是否小数
if CUtils.text_is_decimal(char_1) is False \
or CUtils.text_is_decimal(char_2) is False:
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Tif) \
or CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Bil):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = self.file_info.file_main_name
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def classified(self):
"""
设计国土行业数据的dem_noframe非分幅数据的验证规则
完成 负责人 李宪 在这里检验dem_noframe的识别规则
:return:
"""
super().classified()
file_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name)
check_file_main_name_exist_tif = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
check_file_main_name_exist_img = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Img))
if (not check_file_main_name_exist_tif) and (not check_file_main_name_exist_img):
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Tif) \
or CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Img):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = self.file_info.file_main_name
file_detail_xml = '{0}_21at.xml'.format(self.file_info.file_main_name_with_full_path)
self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def add_file_to_detail_list(self, match_name):
"""
设定国土行业数据三调的附属文件的验证规则（镶嵌影像）
完成 负责人 王学谦 在这里检验三调的附属文件
:return:
"""
file_main_name = self._object_name
file_path = self.file_info.file_path
# 模糊匹配附属文件
if not CUtils.equal_ignore_case(file_path, ''):
match_str = '{0}*xq.*'.format(match_name)
match_file_list = CFile.file_or_dir_fullname_of_path(
file_path, False, match_str, CFile.MatchType_Common)
for file_with_path in match_file_list:
if not CUtils.equal_ignore_case(
CFile.file_main_name(file_with_path),
file_main_name): # 去除自身与同名文件
self.add_file_to_details(file_with_path) # 将文件加入到附属文件列表中
try:
db = CFactory().give_me_db(self.file_info.db_server_id)
metadata_name_before_six_name = db.one_row("select gdstitle from ro_global_dim_space "
"where gdsid = '{0}'".format(match_name)) \
.value_by_name(0, 'gdstitle', None)
metadata_file_name = '{0}{1}.mdb'.format(
match_name, metadata_name_before_six_name)
metadata_file_name_with_path = CFile.join_file(
file_path, metadata_file_name)
if CFile.file_or_path_exist(metadata_file_name_with_path):
self.add_file_to_details(metadata_file_name_with_path)
except:
pass
def search_type(self):
"""
检索插件类型
:return:
"""
listplugin = []
plugins_root_dir = CSys.get_plugins_root_dir()
plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir)
for plugins_type in plugins_type_list:
if CFile.is_dir(CFile.join_file(
plugins_root_dir, plugins_type)) and (
not (str(plugins_type)).startswith('_')):
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), plugins_type)
path = CFile.join_file(CSys.get_plugins_root_dir(),
plugins_type)
plugins_file_list = CFile.file_or_subpath_of_path(
path, '{0}_*.{1}'.format(self.Name_Plugins,
self.FileExt_Py))
for file_name_without_path in plugins_file_list:
file_main_name = CFile.file_main_name(
file_name_without_path)
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, file_main_name, None)
plugins_info = class_classified_obj.get_information()
# 获取插件的类型和名字
plugins_info["dsodid"] = '{0}'.format(
plugins_type) + CFile.unify_seperator + '{0}'.format(
file_main_name)
listplugin.append(plugins_info)
plugin_path = []
# 遍历listplugin
for i in listplugin:
file_dict = {}
# 获取当前文件工作目录
work_path = CFile.file_abs_path('.')
# 拼接通用路径
main_path = work_path + "/imetadata/business/metadata/inbound/plugins/"
# 分割插件类型和名字
list = CUtils.dict_value_by_name(i, "dsodid",
'').split(CFile.unify_seperator)
# 拼接插件所在路径
file_path = main_path + CUtils.dict_value_by_name(
i, "dsodid", '') + "." + self.FileExt_Py
# 格式化文件路径
sorted_file_path = CFile.unify(file_path)
"""
type: dir/file/layer
source: 待压缩的文件路径
target: 压缩后路径和名字(根据用户输入的压缩地址,然后拼接出完整的压缩文件)
"""
file_dict[CResource.Name_Type] = list[0]
file_dict.setdefault(CResource.Name_Source,
[]).append(sorted_file_path)
file_dict[CResource.Name_Target] = str(
CUtils.dict_value_by_name(i, "dsodtype", '')) + ".zip"
print(file_dict)
plugin_path.append(file_dict)
return plugin_path
def parser_file_or_subpath_of_path(self, dataset, ds_id, ds_path,
ds_rule_content, inbound_id):
"""
处理目录(完整路径)下的子目录和文件
:param inbound_id: 入库标识
:param ds_rule_content:
:param dataset: 数据集
:param ds_id: 路径标识
:param ds_path: 路径全名
:return:
"""
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ignore_file_array = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_ignore_file, None)
ignore_dir_array = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_ignore_dir, None)
file_list = CFile.file_or_subpath_of_path(ds_path)
for file_name in file_list:
file_name_with_full_path = CFile.join_file(ds_path, file_name)
if CFile.is_dir(file_name_with_full_path):
CLogger().debug('在目录{0}下发现子目录: {1}'.format(ds_path, file_name))
if CUtils.list_count(ignore_dir_array, file_name) > 0:
CLogger().debug(
'子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name))
continue
path_obj = CDMPathInfo(
self.FileType_Dir, file_name_with_full_path,
dataset.value_by_name(0, 'query_storage_id', ''), None,
ds_id,
dataset.value_by_name(0, 'query_dir_parent_objid', None),
self.get_mission_db_id(), ds_rule_content)
if path_obj.white_black_valid():
path_obj.db_check_and_update(inbound_id)
else:
CLogger().info('目录[{0}]未通过黑白名单检验, 不允许入库! '.format(
file_name_with_full_path))
elif CFile.is_file(file_name_with_full_path):
if CUtils.list_count(ignore_file_array, file_name) > 0:
CLogger().debug(
'子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name))
continue
CLogger().debug('在目录{0}下发现文件: {1}'.format(ds_path, file_name))
file_obj = CDMFileInfo(
self.FileType_File, file_name_with_full_path,
dataset.value_by_name(0, 'query_storage_id', ''), None,
ds_id,
dataset.value_by_name(0, 'query_dir_parent_objid', None),
self.get_mission_db_id(), ds_rule_content)
if file_obj.white_black_valid():
file_obj.db_check_and_update(inbound_id)
else:
CLogger().info('文件[{0}]未通过黑白名单检验, 不允许入库! '.format(
file_name_with_full_path))
def record_as_dict(self) -> dict:
data_record = self._data.record(self.__record_index)
result = dict()
key_list = data_record.keys()
for key in key_list:
result[CUtils.any_2_str(key).lower()] = CUtils.dict_value_by_name(
data_record, key, None, False)
return result
def json_set_attr(cls, json_text, attr_name: str, attr_value) -> str:
new_result = CJson()
rt_json_text = CUtils.any_2_str(json_text)
if not CUtils.equal_ignore_case(rt_json_text, ''):
new_result.load_json_text(rt_json_text)
new_result.set_value_of_name(attr_name, attr_value)
return new_result.to_json()
def join_file(cls, path: str, *paths: AnyStr) -> str:
result = CUtils.any_2_str(path)
for each_path in paths:
real_file_name = CUtils.any_2_str(each_path)
if real_file_name.startswith(r'/') or real_file_name.startswith('\\'):
real_file_name = real_file_name[1:len(real_file_name)]
result = '{0}{1}{2}'.format(result, cls.sep(), real_file_name)
return result
def __inbound_object_detail_of_schema(self, list_file_fullname):
sql_detail_insert = '''
INSERT INTO dm2_storage_obj_detail(
dodid, dodobjectid, dodfilename, dodfileext, dodfilesize,
dodfilecreatetime, dodfilemodifytime,
dodlastmodifytime, dodfiletype)
VALUES (
:dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize,
:dodfilecreatetime, :dodfilemodifytime, now(),
:dodfiletype)
'''
sql_detail_insert_params_list = []
# query_storage_id = self.file_info.storage_id
query_file_relation_name = self.file_info.file_name_with_rel_path
for item_file_name_with_path in list_file_fullname:
CLogger().debug(item_file_name_with_path)
if not CFile.file_or_path_exist(item_file_name_with_path):
continue
params = dict()
file_relation_name = CFile.file_relation_path(
item_file_name_with_path, self.file_info.root_path)
if CUtils.equal_ignore_case(query_file_relation_name,
file_relation_name):
params['dodid'] = self.object_id
else:
params['dodid'] = CUtils.one_id()
# 文件类型
params['dodfiletype'] = self.FileType_File
if CFile.is_dir(item_file_name_with_path):
params['dodfiletype'] = self.FileType_Dir
params['dodobjectid'] = self.object_id
params['dodfilename'] = CFile.unify(file_relation_name)
params['dodfileext'] = CFile.file_ext(item_file_name_with_path)
params['dodfilesize'] = CFile.file_size(item_file_name_with_path)
params['dodfilecreatetime'] = CFile.file_create_time(
item_file_name_with_path)
params['dodfilemodifytime'] = CFile.file_modify_time(
item_file_name_with_path)
# params['dodstorageid'] = query_storage_id
# params['dodfilerelationname'] = CFile.file_relation_path(
# item_file_name_with_path,
# self.file_info.root_path)
sql_params_tuple = (sql_detail_insert, params)
sql_detail_insert_params_list.append(sql_params_tuple)
if len(sql_detail_insert_params_list) > 0:
try:
CFactory().give_me_db(
self.file_info.db_server_id).execute_batch(
sql_detail_insert_params_list)
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(
error.__str__()))
return CResult.merge_result(self.Failure, '处理失败!')
return CResult.merge_result(self.Success, '处理完毕!')
def check_all_ib_file_or_path_existed(self, ib_id):
"""
判断待入库数据的元数据, 与实体数据是否相符
. 返回CResult
. 如果全部相符, 则返回True
. 如果有任何一个不符, 则返回False, 且把不符的文件名通过信息返回
:param ib_id:
:return:
"""
invalid_file_list = []
more_failure_file = False
sql_all_ib_file = '''
select
coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) || dm2_storage_file.dsffilerelationname as file_name
, dm2_storage_file.dsffilesize as file_size
, dm2_storage_file.dsffilemodifytime as file_modify_time
from dm2_storage_file left join dm2_storage
on dm2_storage.dstid = dm2_storage_file.dsfstorageid
where dsf_ib_id = :ib_id
'''
params_all_ib_file = {'ib_id': ib_id}
ds_ib_file = CFactory().give_me_db(self.get_mission_db_id()).all_row(
sql_all_ib_file, params_all_ib_file)
for ds_ib_file_index in range(ds_ib_file.size()):
file_valid = True
file_name = ds_ib_file.value_by_name(ds_ib_file_index, 'file_name',
'')
if not CUtils.equal_ignore_case(file_name, ''):
if not CFile.file_or_path_exist(file_name):
file_valid = False
elif not CUtils.equal_ignore_case(
CFile.file_modify_time(file_name),
ds_ib_file.value_by_name(ds_ib_file_index,
'file_modify_time', '')):
file_valid = False
elif CFile.file_size(file_name) != ds_ib_file.value_by_name(
ds_ib_file_index, 'file_size', 0):
file_valid = False
if not file_valid:
if len(invalid_file_list) <= 3:
invalid_file_list.append(file_name)
else:
more_failure_file = True
break
if len(invalid_file_list) > 0:
message = ''
for invalid_file in invalid_file_list:
message = CUtils.str_append(message, invalid_file)
if more_failure_file:
message = CUtils.str_append(message, '...')
message = CUtils.str_append(message, '上述数据与库中记录不统一, 请重新扫描入库! ')
return CResult.merge_result(self.Failure, message)
else:
return CResult.merge_result(self.Success, '所有文件均存在, 且与库中记录统一! ')
def register_dm_metadata_plugins(self):
sql_register_dm_metadata_plugins_clear = '''
truncate table dm2_storage_object_def cascade
'''
sql_unregister_dm_metadata_plugins = '''
delete from dm2_storage_object_def where dsodid = :dsodid
'''
sql_register_dm_metadata_plugins = '''
insert into dm2_storage_object_def(
dsodid, dsodtitle, dsodtype, dsodtypetitle, dsodtypecode, dsodgroup, dsodgrouptitle,
dsodcatalog, dsodcatalogtitle, dsod_otheroption)
values (:dsodid, :dsodtitle, :dsodtype, :dsodtypetitle, :dsodtypecode, :dsodgroup, :dsodgrouptitle,
:dsodcatalog, :dsodcatalogtitle, :dsod_otheroption)
'''
CFactory().give_me_db().execute(sql_register_dm_metadata_plugins_clear)
plugins_root_dir = CSys.get_plugins_root_dir()
plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir)
for plugins_type in plugins_type_list:
if CFile.is_dir(CFile.join_file(
plugins_root_dir, plugins_type)) and (
not (str(plugins_type)).startswith('_')):
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), plugins_type)
path = CFile.join_file(CSys.get_plugins_root_dir(),
plugins_type)
plugins_file_list = CFile.file_or_subpath_of_path(
path, '{0}_*.{1}'.format(self.Name_Plugins,
self.FileExt_Py))
for file_name_without_path in plugins_file_list:
file_main_name = CFile.file_main_name(
file_name_without_path)
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, file_main_name, None)
plugins_info = class_classified_obj.get_information()
json_obj = CJson()
json_obj.set_value_of_name(
self.Name_Is_Spatial,
CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Is_Spatial,
False))
json_obj.set_value_of_name(
self.Name_Is_DataSet,
CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Is_Dataset,
False))
plugins_info['dsod_otheroption'] = json_obj.to_json()
print('{0}/{1}:{2}'.format(plugins_type, file_main_name,
plugins_info))
CFactory().give_me_db().execute(
sql_unregister_dm_metadata_plugins, plugins_info)
CFactory().give_me_db().execute(
sql_register_dm_metadata_plugins, plugins_info)
def classified(self):
"""
设计国土行业数据guoqing_frame的验证规则（国情影像—分幅影像）
完成 负责人 王学谦 在这里检验guoqing_frame的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext # 初始化需要的参数
file_path = self.file_info.file_path
file_object_name = file_main_name[:]
if len(file_main_name) >= 21: # 本类文件默认至少为20位
file_object_name = file_main_name[:20] # 截取前20位
elif len(file_main_name) == 20: # 20位基本为附属文件
pass
else:
return self.Object_Confirm_IUnKnown, self._object_name
match_str = '(?i)^' + file_object_name + r'[a-zA-Z][.]tif'
check_file_main_name_exist = \
CFile.find_file_or_subpath_of_path(file_path, match_str, CFile.MatchType_Regex)
if not check_file_main_name_exist: # 检查主文件存在性
return self.Object_Confirm_IUnKnown, self._object_name
# 文件名第1，4，11，12，16，21位为字母，第2，3，5-10，14，15，17-20位是数字
name_sub_1 = file_main_name[0:1]
name_sub_2_to_3 = file_main_name[1:3]
name_sub_4 = file_main_name[3:4]
name_sub_5_to_10 = file_main_name[4:10]
name_sub_11_to_12 = file_main_name[10:12]
name_sub_14_to_15 = file_main_name[13:15]
name_sub_16 = file_main_name[15:16]
name_sub_17_to_20 = file_main_name[16:20]
name_sub_21 = file_main_name[20:21]
if CUtils.text_is_alpha(name_sub_1) is False \
or CUtils.text_is_numeric(name_sub_2_to_3) is False \
or CUtils.text_is_alpha(name_sub_4) is False \
or CUtils.text_is_numeric(name_sub_5_to_10) is False \
or CUtils.text_is_alpha(name_sub_11_to_12) is False \
or CUtils.text_is_numeric(name_sub_14_to_15) is False \
or CUtils.text_is_alpha(name_sub_16) is False \
or CUtils.text_is_numeric(name_sub_17_to_20) is False \
or CUtils.text_is_alpha(name_sub_21) is False:
return self.Object_Confirm_IUnKnown, self._object_name
if len(file_main_name) == 21 and CUtils.equal_ignore_case(
file_ext, 'tif'):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
self.add_file_to_detail_list(file_object_name) # 在这里设置不同名的附属文件
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def metadata_bus_dict_process_custom(self, metadata_bus_dict):
super().metadata_bus_dict_process_custom(metadata_bus_dict)
centertime = CUtils.dict_value_by_name(metadata_bus_dict, 'centertime', None)
publishdate = CUtils.dict_value_by_name(metadata_bus_dict, 'publishdate', None)
if not CUtils.equal_ignore_case(centertime, ''):
centertime = centertime[0:8] + ' ' + centertime[8:]
metadata_bus_dict['centertime'] = centertime
if not CUtils.equal_ignore_case(publishdate, ''):
publishdate = publishdate[0:8] + ' ' + publishdate[8:]
metadata_bus_dict['publishdate'] = publishdate
def classified(self):
file_path = self.file_info.file_path
file_ext = self.file_info.file_ext
if CUtils.text_match_re(file_path, r'(?i)\d{4}.{2}[\\\\/]FenFu[\\\\/]' + self.get_coordinate_system_title()) \
and CUtils.equal_ignore_case(file_ext, 'shp'):
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
else:
return super().classified()
def __condition_list_2_sql(self, field, value_list, quoted_value):
if quoted_value:
in_sql = CUtils.list_2_str(value_list, "'", ", ", "'", True)
else:
in_sql = CUtils.list_2_str(value_list, "", ", ", "", True)
if CUtils.equal_ignore_case(in_sql, ''):
return ""
else:
return "{0} in ({1})".format(field, in_sql)
def metadata_bus_dict_process_custom(self, metadata_bus_dict):
"""
对部分需要进行运算的数据进行处理
"""
super().metadata_bus_dict_process_custom(metadata_bus_dict)
productattribute = CUtils.dict_value_by_name(metadata_bus_dict,
'productattribute', None)
if not CUtils.equal_ignore_case(productattribute, ''):
if len(productattribute) > 2:
metadata_bus_dict['productattribute'] = productattribute[:2]
