def get_metadata_bus_filename_by_file(self) -> str:
"""
卫星数据解压后, 哪个文件是业务元数据?
:return:
"""
return CFile.join_file(
self.file_content.content_root_dir,
self.get_fuzzy_metadata_file(
'H1C_OPER_.*.meta.(xml|XML)',
'{0}.meta.xml'.format(self.classified_object_name())))
def get_metadata_bus_filename_by_file(self) -> str:
"""
卫星数据解压后, 哪个文件是业务元数据?
:return:
"""
return CFile.join_file(
self.file_content.content_root_dir,
self.get_fuzzy_metadata_file(
'.*DIM.*_P_.*.XML',
'{0}.xml'.format(self.classified_object_name()), True))
def get_metadata_bus_filename_by_file(self) -> str:
"""
卫星数据解压后, 哪个文件是业务元数据?
:return:
"""
return CFile.join_file(
self.file_content.content_root_dir,
self.get_fuzzy_metadata_file(
'ZY30(1|2).TMS.*(NAD|MUX).xml',
'{0}.xml'.format(self.classified_object_name())))
def get_metadata_bus_filename_by_file(self) -> str:
"""
卫星数据解压后, 哪个文件是业务元数据?
:return:
"""
return CFile.join_file(
self.file_content.content_root_dir,
self.get_fuzzy_metadata_file(r'(?i).*PAN\d[.]xml', '{0}-PAN.xml'.format(self.classified_object_name())
)
)
def get_metadata_bus_filename_by_file(self) -> str:
"""
卫星数据解压后, 哪个文件是业务元数据?
:return:
"""
return CFile.join_file(
self.file_content.content_root_dir,
self.get_fuzzy_metadata_file(
'GF701.*_.*_.*_[0-9]{10}.xml',
'{0}.xml'.format(self.classified_object_name())))
def get_transform_file_to_metadata_view(self, parser: CMetaDataParser):
"""
对于需要由图像文件转换为预览图文件的方式进行处理
"""
return CFile.join_file(
self.get_sat_file_originally_path(),
self.get_fuzzy_metadata_file(
r'(?i).*_browse[.]tif$',
'{0}_browse.tif'.format(self.classified_object_name())
)
)
def classified(self):
"""
设计国土行业数据的dem_10_dem验证规则
完成 负责人 李宪 在这里检验dem_10_dem的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
check_file_main_name_length = len(file_main_name) == 13
if not check_file_main_name_length:
return self.Object_Confirm_IUnKnown, self._object_name
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_main_name)
check_file_main_name_exist_tif = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
check_file_main_name_exist_bil = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Bil))
if (not check_file_main_name_exist_tif) and (
not check_file_main_name_exist_bil):
return self.Object_Confirm_IUnKnown, self._object_name
"""
下面判别第1位是字母
下面判别第4位是字母
下面判别第23位是数字
下面判别第567位是数字
下面判别第8910位是数字
下面判别第111213位是DOM
"""
char_1 = file_main_name[0:1]
char_2_3 = file_main_name[1:3]
char_4 = file_main_name[3:4]
char_5_to_7 = file_main_name[4:7]
char_8_to_10 = file_main_name[7:10]
char_11_to_13 = file_main_name[10:13]
if CUtils.text_is_alpha(char_1) is False \
or CUtils.text_is_numeric(char_2_3) is False \
or CUtils.text_is_alpha(char_4) is False \
or CUtils.text_is_numeric(char_5_to_7) is False \
or CUtils.text_is_numeric(char_8_to_10) is False \
or CUtils.equal_ignore_case(char_11_to_13, "DEM") is False:
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \
or CUtils.equal_ignore_case(file_ext, self.Name_Bil):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 王学谦
:param parser:
:return:
"""
super().qa_file_custom(parser)
file_object_name_list = re.findall(r'(?i)^([a-z]{2}\S+\d{4}[01]\d[0123]\d)[a-z][-]\d+$',
self.file_info.file_main_name)
file_object_name = file_object_name_list[0] # 去除尾部的F/M/P-数字
metadata_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name)
check_file_metadata_bus_exist = False
ext = self.Transformer_XML
temp_metadata_bus_file_Y = '{0}Y.xml'.format(metadata_main_name_with_path)
temp_metadata_bus_file_P = '{0}P.xml'.format(metadata_main_name_with_path)
temp_metadata_bus_file_M = '{0}M.xml'.format(metadata_main_name_with_path)
if CFile.file_or_path_exist(temp_metadata_bus_file_Y): # 存在Y-P-M的优先级
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = temp_metadata_bus_file_Y
elif CFile.file_or_path_exist(temp_metadata_bus_file_P):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = temp_metadata_bus_file_P
elif CFile.file_or_path_exist(temp_metadata_bus_file_M):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = temp_metadata_bus_file_M
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Error,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本文件缺少业务元数据'
}
)
else:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: self.metadata_bus_src_filename_with_path,
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path)
}
)
def classified(self):
"""
设计国土行业数据mosaic的验证规则（镶嵌影像）
完成 负责人 王学谦 在这里检验mosaic的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext # 初始化需要的参数
file_name_with_full_path = self.file_info.file_name_with_full_path
file_object_name = file_main_name[:] # 主要名称截取
if file_name_with_full_path.endswith('_21at.xml'): # 元数据文件的情况
if len(file_main_name) > 5:
file_object_name = file_main_name[:-5]
else:
return self.Object_Confirm_IUnKnown, self._object_name
else: # 矢量文件的情况
xq_list = [
'xq.shp', 'xq.shx', 'xq.dbf', 'xq.sbx', 'xq.prj', 'xq.sbn'
]
for xq_end in xq_list:
if file_name_with_full_path.lower().endswith(xq_end):
if len(file_main_name) > 2:
file_object_name = file_main_name[:-2]
break
else:
return self.Object_Confirm_IUnKnown, self._object_name
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_object_name)
check_file_main_name_exist = \
CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) or \
CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Img))
if not check_file_main_name_exist: # 检查主文件存在性
return self.Object_Confirm_IUnKnown, self._object_name
check_file_shp_exist = CFile.file_or_path_exist(
'{0}xq.shp'.format(file_main_name_with_path))
if not check_file_shp_exist: # 检查矢量文件存在性
return self.Object_Confirm_IUnKnown, self._object_name
# 检查后缀名
if CUtils.equal_ignore_case(file_ext,
self.Name_Tif) or CUtils.equal_ignore_case(
file_ext, self.Name_Img):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
self.add_file_to_detail_list()
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def init_before_test(self):
plugins_info = self.create_plugins().get_information()
plugins_catalog = CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Catalog_Title, '')
plugins_type = CUtils.dict_value_by_name(plugins_info,
CPlugins.Plugins_Info_Type,
'')
self._test_file_root_path = settings.application.xpath_one(
self.Path_Setting_Dir_Test_Data, '')
self._test_file_parent_path = CFile.join_file(
settings.application.xpath_one(self.Path_Setting_Dir_Test_Data,
''), plugins_catalog)
def plugins(cls, file_info: CDMFilePathInfoEx,
plugins_id: str) -> CPlugins:
"""
根据给定的文件信息和插件名称, 直接创建插件对象
如果插件文件不存在, 则返回None
:param file_info:
:param plugins_id:
:return:
"""
target_type = file_info.file_type
target_path = CFile.join_file(CSys.get_plugins_root_dir(), target_type)
plugins_file_name = CFile.join_file(
target_path, '{0}.{1}'.format(plugins_id, cls.FileExt_Py))
if not CFile.file_or_path_exist(plugins_file_name):
return None
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), target_type)
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, plugins_id, file_info)
return class_classified_obj
def init_qa_file_list(self, parser: CMetaDataParser) -> list:
"""
设定镶嵌影像的质检列表,调用默认的主文件质检列表方法，并拼接剩余附属文件
完成 负责人 王学谦 在这里检验设定镶嵌影像的质检列表
"""
file_main_name = self.classified_object_name()
list_qa = list()
list_qa.extend(
self.init_qa_file_integrity_default_list(
self.file_info.file_name_with_full_path)) # 调用默认的规则列表
# 获取xq.shp数据并判断完整性
xq_shp_with_full_path = CFile.join_file(
self.file_info.file_path, '{0}xq.shp'.format(file_main_name))
list_qa.extend(
self.init_qa_file_integrity_default_list(xq_shp_with_full_path))
# list_qa.extend([
# {
# self.Name_FileName: '{0}xq.shp'.format(file_main_name),
# self.Name_ID: 'shp',
# self.Name_Title: 'shp文件',
# self.Name_Group: self.QA_Group_Data_Integrity,
# self.Name_Result: self.QA_Result_Error,
# self.Name_Format: self.DataFormat_Vector_File
# }, {
# self.Name_FileName: '{0}xq.shx'.format(file_main_name),
# self.Name_ID: 'shx',
# self.Name_Title: 'shx文件',
# self.Name_Group: self.QA_Group_Data_Integrity,
# self.Name_Result: self.QA_Result_Error
# }, {
# self.Name_FileName: '{0}xq.dbf'.format(file_main_name),
# self.Name_ID: 'dbf',
# self.Name_Title: 'dbf文件',
# self.Name_Group: self.QA_Group_Data_Integrity,
# self.Name_Result: self.QA_Result_Error
# }, {
# self.Name_FileName: '{0}xq.sbx'.format(file_main_name),
# self.Name_ID: 'sbx',
# self.Name_Title: 'sbx文件',
# self.Name_Group: self.QA_Group_Data_Integrity,
# self.Name_Result: self.QA_Result_Warn
# }, {
# self.Name_FileName: '{0}xq.prj'.format(file_main_name),
# self.Name_ID: 'prj',
# self.Name_Title: 'prj文件',
# self.Name_Group: self.QA_Group_Data_Integrity,
# self.Name_Result: self.QA_Result_Warn
# }
# ])
return list_qa
def get_custom_affiliated_file_character(self):
file_path = self.file_info.file_path
letter_location = file_path.find('镶嵌影像成果')
shp_path = CFile.join_file(file_path[:letter_location - 1], '影像时相接边图', self.get_yuji())
shp_regularexpression = '(?i)^.*_.*_' + self.get_coordinate_system() + r'\.shp$'
return [
{
self.Name_FilePath: shp_path, # 附属文件的路径
self.Name_RegularExpression: shp_regularexpression, # 附属文件的匹配规则
# 应该从上面匹配到的文件剔除的文件的匹配规则
self.Name_No_Match_RegularExpression: None
}
]
def process(self) -> str:
"""
注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!!
:return:
"""
result_process = self.process_vector()
if CResult.result_success(result_process):
file_path = self.file_content.work_root_dir
dict_temp_file_name = {
self.Name_Native_Center:
'{0}_native_center.wkt'.format(self.object_name),
self.Name_Native_BBox:
'{0}_native_bbox.wkt'.format(self.object_name),
self.Name_Native_Geom:
'{0}_native_geom.wkt'.format(self.object_name),
self.Name_Wgs84_Center:
'{0}_wgs84_center.wkt'.format(self.object_name),
self.Name_Wgs84_BBox:
'{0}_wgs84_bbox.wkt'.format(self.object_name),
self.Name_Wgs84_Geom:
'{0}_wgs84_geom.wkt'.format(self.object_name)
}
dict_temp_prj_name = {
self.Name_Prj_Wkt:
CResult.result_info(result_process, self.Name_Prj_Wkt, None),
self.Name_Prj_Proj4:
CResult.result_info(result_process, self.Name_Prj_Proj4, None),
self.Name_Prj_Project:
CResult.result_info(result_process, self.Name_Prj_Project,
None),
self.Name_Prj_Coordinate:
CResult.result_info(result_process, self.Name_Prj_Coordinate,
None),
self.Name_Prj_Source:
CResult.result_info(result_process, self.Name_Prj_Source,
None),
self.Name_Prj_Zone:
CResult.result_info(result_process, self.Name_Prj_Zone, None),
self.Name_Prj_Degree:
CResult.result_info(result_process, self.Name_Prj_Degree, None)
}
result = CResult.merge_result(self.Success, '处理完毕!')
for file_type, file_name in dict_temp_file_name.items():
result = CResult.merge_result_info(
result, file_type, CFile.join_file(file_path, file_name))
for prj_type, prj_name in dict_temp_prj_name.items():
result = CResult.merge_result_info(result, prj_type, prj_name)
else:
result = CResult.merge_result(
self.Failure, CResult.result_message(result_process))
return result
def classified(self):
"""
设计国土行业数据的dom-10验证规则
完成 负责人 李宪 在这里检验dem-10的元数据文件格式时, 应该一个一个类型的对比, 找到文件时, 将该文件的格式和文件名存储到类的私有属性中, 以便在元数据处理时直接使用
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
check_file_main_name_length = len(file_main_name) == 10
if not check_file_main_name_length:
return self.Object_Confirm_IUnKnown, self._object_name
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_main_name)
check_file_main_name_exist_tif = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
check_file_main_name_exist_bil = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Bil))
if (not check_file_main_name_exist_tif) and (
not check_file_main_name_exist_bil):
return self.Object_Confirm_IUnKnown, self._object_name
"""
下面判别第1位是字母
下面判别第4位是字母
下面判别第23位是数字
下面判别第567位是数字
下面判别第8910位是数字
"""
char_1 = file_main_name[0:1]
char_2_3 = file_main_name[1:3]
char_4 = file_main_name[3:4]
char_5_to_7 = file_main_name[4:7]
char_8_to_10 = file_main_name[7:10]
if CUtils.text_is_alpha(char_1) is False \
or CUtils.text_is_numeric(char_2_3) is False \
or CUtils.text_is_alpha(char_4) is False \
or CUtils.text_is_numeric(char_5_to_7) is False \
or CUtils.text_is_numeric(char_8_to_10) is False:
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \
or CUtils.equal_ignore_case(file_ext, self.Name_Bil):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def __plugins_classified_of_directory__(
cls, file_info: CDMFilePathInfoEx) -> CPlugins:
"""
使用系统目录下的所有插件进行识别
:param file_info:
:return:
"""
target = file_info.file_main_name
target_type = file_info.file_type
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), target_type)
path = CFile.join_file(CSys.get_plugins_root_dir(), target_type)
plugins_file_list = CFile.file_or_subpath_of_path(
path, '{0}_*.{1}'.format(cls.Name_Plugins, cls.FileExt_Py))
for file_name_without_path in plugins_file_list:
file_main_name = CFile.file_main_name(file_name_without_path)
try:
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, file_main_name, file_info)
obj_info = class_classified_obj.get_information()
# 王西亚 添加逻辑于2021-02-12, 解决项目插件调试不方便的问题
obj_classified_valid = False
obj_owner_project_id = CUtils.dict_value_by_name(
obj_info, CPlugins.Plugins_Info_Project_ID, None)
system_project_id = settings.application.xpath_one(
cls.Path_Setting_Project_ID, None)
if obj_owner_project_id is None:
obj_classified_valid = True
else:
obj_classified_valid = CUtils.equal_ignore_case(
obj_owner_project_id, system_project_id)
if obj_classified_valid:
object_confirm, object_name = class_classified_obj.classified(
)
if object_confirm != cls.Object_Confirm_IUnKnown:
obj_id = class_classified_obj.get_id()
CLogger().debug('{0} is classified as {1}.{2}'.format(
target, obj_info, obj_id))
return class_classified_obj
except Exception as error:
CLogger().debug('插件[{0}]解析出现异常, 错误信息为: [{1}], 请检查!'.format(
file_main_name, error.__str__()))
if settings.application.xpath_one(
'{0}.{1}'.format(cls.Name_Application, cls.Name_Debug),
cls.DB_False) == cls.DB_True:
raise
else:
continue
else:
return None
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 李宪
:param parser:
:return:
"""
super().qa_file_custom(parser)
file_metadata_name_with_path = CFile.join_file(
self.file_info.file_path, self.file_info.file_main_name)
check_file_metadata_name_exist = False
ext_list = ['xls', 'xlsx', 'mat', 'mdb']
for ext in ext_list:
temp_metadata_bus_file = '{0}.{1}'.format(
file_metadata_name_with_path, ext)
if CFile.file_or_path_exist(temp_metadata_bus_file):
check_file_metadata_name_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = temp_metadata_bus_file
break
if not check_file_metadata_name_exist:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
'',
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Error,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'本文件缺少业务元数据'
})
else:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
self.metadata_bus_src_filename_with_path,
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Pass,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path)
})
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 王学谦
:param parser:
:return:
"""
super().qa_file_custom(parser)
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
check_file_metadata_bus_exist = False
ext = self.Transformer_XML
metadata_name_with_path = CFile.join_file(
file_path, '{0}M.xml'.format(file_main_name[:-1]))
if CFile.file_or_path_exist(metadata_name_with_path):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = metadata_name_with_path
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
'',
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Error,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'本文件缺少业务元数据'
})
else:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
self.metadata_bus_src_filename_with_path,
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Pass,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'业务元数据[{0}]存在'.format(
CFile.file_name(self.metadata_bus_src_filename_with_path))
})
def process(self) -> str:
"""
完成 负责人 赵宇飞 在这里提取影像数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式
注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!!
:return:
"""
out_metadata_file_fullname = CFile.join_file(self.file_content.work_root_dir, self.FileName_MetaData)
raster_mdreader = CRasterMDReader(self.file_info.file_name_with_full_path)
# result = raster_mdreader.get_metadata_2_file(out_metadata_file_fullname)
result = CProcessUtils.processing_method(raster_mdreader.get_metadata_2_file, out_metadata_file_fullname)
# 进程调用模式
# p_one = Process(target=raster_mdreader.get_metadata_2_file, args=(out_metadata_file_fullname,))
# p_one.start()
# p_one.join()
return CResult.merge_result_info(result, self.Name_Format, self.MetaDataFormat_Json)
def process_mission(self, dataset) -> str:
"""
:param dataset:
:return:
"""
ds_na_id = dataset.value_by_name(0, 'na_id', '')
ds_app_id = dataset.value_by_name(0, 'app_id', '')
ds_object_id = dataset.value_by_name(0, 'object_id', '')
ds_object_type = dataset.value_by_name(0, 'object_type', '')
ds_object_name = dataset.value_by_name(0, 'object_name', '')
ds_object_access = dataset.value_by_name(0, 'object_access',
self.DataAccess_Forbid)
CLogger().debug('与第三方模块[{0}]同步的对象为: [{1}]'.format(
ds_app_id, ds_object_name))
try:
module_file_name = CFile.join_file(
CSys.get_metadata_data_access_modules_root_dir(),
'{0}.{1}'.format(ds_app_id, self.FileExt_Py))
if not CFile.file_or_path_exist(module_file_name):
message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id)
result = CResult.merge_result(self.Success, message)
self.update_sync_result(ds_na_id, result)
return result
module_obj = CObject.create_module_instance(
CSys.get_metadata_data_access_modules_root_name(), ds_app_id,
self.get_mission_db_id())
if module_obj is None:
message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id)
result = CResult.merge_result(self.Success, message)
self.update_sync_result(ds_na_id, result)
return result
module_title = CUtils.dict_value_by_name(module_obj.information(),
self.Name_Title, '')
result = module_obj.sync(ds_object_access, ds_object_id,
ds_object_name, ds_object_type, None)
self.update_sync_result(ds_na_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure,
'与第三方模块[{0}]同步的对象: [{1}]的同步过程出现异常, 详细情况: [{2}]!'.format(
ds_app_id, ds_object_name, error.__str__()))
self.update_sync_result(ds_na_id, result)
return result
def process(self) -> str:
"""
在这里提取矢量数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式
:return:
"""
result = super().process()
metadata_main_name_with_path = CFile.join_file(
self.file_info.file_path, self.file_info.file_main_name)
metadata_filename = '{0}.xml'.format(metadata_main_name_with_path[:-2])
format_metadata = self.MetaDataFormat_XML
result = CResult.merge_result_info(result, self.Name_FileName,
metadata_filename)
return CResult.merge_result_info(result, self.Name_Format,
format_metadata)
def classified(self):
"""
设计国土行业数据的dom-12验证规则
完成 负责人 李宪 在这里检验dom-12的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
check_file_main_name_length = len(file_main_name) == 12
if not check_file_main_name_length:
return self.Object_Confirm_IUnKnown, self._object_name
file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name)
check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
if not check_file_main_name_exist:
return self.Object_Confirm_IUnKnown, self._object_name
"""
下面判别第1位是字母
下面判别第4位是字母
下面判别第23位是数字
下面判别第567位是数字
下面判别第8910位是数字
"""
char_1 = file_main_name[0:1]
char_2_3 = file_main_name[1:3]
char_4 = file_main_name[3:4]
char_5_to_7 = file_main_name[4:7]
char_8_to_12 = file_main_name[7:12]
if CUtils.text_is_alpha(char_1) is False \
or CUtils.text_is_numeric(char_2_3) is False \
or CUtils.text_is_alpha(char_4) is False \
or CUtils.text_is_numeric(char_5_to_7) is False \
or CUtils.text_is_numeric(char_8_to_12) is False:
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 王学谦
:param parser:
:return:
"""
super().qa_file_custom(parser)
metadata_main_name_with_path = CFile.join_file(
self.file_info.file_path, self.file_info.file_main_name)
check_file_metadata_bus_exist = False
temp_metadata_bus_file = '{0}.xml'.format(
metadata_main_name_with_path[:-2])
if CFile.file_or_path_exist(temp_metadata_bus_file):
check_file_metadata_bus_exist = True
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
'',
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Error,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'本文件缺少业务元数据'
})
else:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
self.metadata_bus_src_filename_with_path,
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Pass,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path)
})
def is_raster_file_integrity(cls, raster_file_with_path: str) -> bool:
"""
判断影像数据的文件完整性，img
xxx.img 文件 栅格数据可读 错误
xxx.ige 文件 img小于1M时必须存在 警告
@param raster_file_with_path:
@return:
"""
file_ext = CFile.file_ext(raster_file_with_path)
if file_ext.lower() == 'img':
size = CFile.file_size(raster_file_with_path)
if size < 1024 * 1024:
file_main_name = CFile.file_main_name(raster_file_with_path)
file_path = CFile.file_path(raster_file_with_path)
ige = CFile.join_file(file_path, file_main_name + '.ige')
if not CFile.file_or_path_exist(ige):
return False
return True
def classified(self):
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext # 初始化需要的参数
file_object_name = file_main_name[:]
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_object_name)
if CUtils.equal_ignore_case(file_ext, self.Name_Shp):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
if CFile.file_or_path_exist('{0}.{1}'.format(
file_main_name_with_path, self.Name_Shp)):
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
else:
self._object_confirm = self.Object_Confirm_IUnKnown
self._object_name = None
return self._object_confirm, self._object_name
def get_custom_affiliated_file_character(self):
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
regularexpression = '(?i)^' + file_main_name[:-1] + r'.\..*'
letter_location = file_path.find('FenFu')
shp_path = CFile.join_file(file_path[:letter_location - 1], 'FenFu', self.get_coordinate_system_title())
return [
{
self.Name_FilePath: file_path, # 附属文件的路径
self.Name_RegularExpression: regularexpression, # 附属文件的匹配规则
# 应该从上面匹配到的文件剔除的文件的匹配规则
self.Name_No_Match_RegularExpression: '(?i)^' + file_main_name + r'\..*$'
}, {
self.Name_FilePath: shp_path, # 附属文件的路径
self.Name_RegularExpression: r'(?i)\.shp$', # 附属文件的匹配规则
# 应该从上面匹配到的文件剔除的文件的匹配规则
self.Name_No_Match_RegularExpression: None
}
]
def db_update_status_on_path_invalid(self):
"""
处理目录不存在时的业务
1. 标记已经入库的文件为无效
1. 标记已经入库的子目录为无效
:return:
"""
path_name_with_relation_path = CFile.join_file(
self.file_path_with_rel_path, '')
params = dict()
params['dsdStorageID'] = self.storage_id
params['dsdSubDirectory'] = path_name_with_relation_path
sql_update_file_invalid = '''
update dm2_storage_file
set dsffilevalid = 0, dsfscanstatus = 0
where dsfdirectoryid in (
select dsdid
from dm2_storage_directory
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1
)
'''.format(CFile.sep())
sql_update_path_invalid = '''
update dm2_storage_directory
set dsd_directory_valid = 0, dsdscanstatus = 0, dsdscanfilestatus = 0, dsdscandirstatus = 0
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1
'''.format(CFile.sep())
engine = CFactory().give_me_db(self.db_server_id)
session = engine.give_me_session()
try:
engine.session_execute(session, sql_update_file_invalid, params)
engine.session_execute(session, sql_update_path_invalid, params)
engine.session_commit(session)
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
engine.session_rollback(session)
finally:
engine.session_close(session)
def get_custom_affiliated_file_character(self):
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
regularexpression = '(?i)^' + file_main_name[:-1] + r'.\..*'
letter_location_FenFu = file_path.find('FenFu')
file_path_sub = file_path[letter_location_FenFu + 6:]
letter_location = file_path_sub.find(CFile.sep())
shp_path = CFile.join_file(file_path[:letter_location_FenFu + 5], file_path_sub[:letter_location])
return [
{
self.Name_FilePath: file_path, # 附属文件的路径
self.Name_RegularExpression: regularexpression, # 附属文件的匹配规则
# 应该从上面匹配到的文件剔除的文件的匹配规则
self.Name_No_Match_RegularExpression: '(?i)^' + file_main_name + r'\..*$'
}, {
self.Name_FilePath: shp_path, # 附属文件的路径
self.Name_RegularExpression: r'(?i)\.shp$', # 附属文件的匹配规则
# 应该从上面匹配到的文件剔除的文件的匹配规则
self.Name_No_Match_RegularExpression: None
}
]
def test_metadata_time(self):
self.init_before_test() # 初始化路径
for test_file_info in self.test_file_info_list():
file_type, test_file_with_rel_path, correct_object_confirm, correct_object_name = \
self.get_test_file_info(test_file_info)
if correct_object_confirm == self.Object_Confirm_IKnown:
test_file_with_full_path = CFile.join_file(self._test_file_parent_path, test_file_with_rel_path)
# 获取插件对象
file_info, plugins_obj, metadata_parser = self.get_test_obj(file_type, test_file_with_full_path)
# 执行测试
plugins_obj.parser_metadata_with_qa(metadata_parser)
plugins_obj.parser_metadata_time_after_qa(metadata_parser)
# 获取结果
result_with_time, message_with_time, time_information = metadata_parser.metadata.metadata_time()
plugins_obj.destroy_virtual_content()
# 录入测试信息
allure.attach('处理信息为{0},时间元数据内容为{1}'
.format(message_with_time, time_information),
'{0}'.format(test_file_with_rel_path),
allure.attachment_type.TEXT)
assert result_with_time
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 李宪
:param parser:
:return:
"""
metadata_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name)
check_file_metadata_bus_exist = False
ext = self.Transformer_XML
temp_metadata_bus_file = '{0}_21at.xml'.format(metadata_main_name_with_path)
if CFile.file_or_path_exist(temp_metadata_bus_file):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = temp_metadata_bus_file
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本文件缺少业务元数据，是正常现象'
}
)
else:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: self.metadata_bus_src_filename_with_path,
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path)
}
)