def get_metadata_bus_filename_by_file(self) -> str: """ 卫星数据解压后, 哪个文件是业务元数据? :return: """ return CFile.join_file( self.file_content.content_root_dir, self.get_fuzzy_metadata_file( 'H1C_OPER_.*.meta.(xml|XML)', '{0}.meta.xml'.format(self.classified_object_name())))
def get_metadata_bus_filename_by_file(self) -> str: """ 卫星数据解压后, 哪个文件是业务元数据? :return: """ return CFile.join_file( self.file_content.content_root_dir, self.get_fuzzy_metadata_file( '.*DIM.*_P_.*.XML', '{0}.xml'.format(self.classified_object_name()), True))
def get_metadata_bus_filename_by_file(self) -> str: """ 卫星数据解压后, 哪个文件是业务元数据? :return: """ return CFile.join_file( self.file_content.content_root_dir, self.get_fuzzy_metadata_file( 'ZY30(1|2).TMS.*(NAD|MUX).xml', '{0}.xml'.format(self.classified_object_name())))
def get_metadata_bus_filename_by_file(self) -> str: """ 卫星数据解压后, 哪个文件是业务元数据? :return: """ return CFile.join_file( self.file_content.content_root_dir, self.get_fuzzy_metadata_file(r'(?i).*PAN\d[.]xml', '{0}-PAN.xml'.format(self.classified_object_name()) ) )
def get_metadata_bus_filename_by_file(self) -> str: """ 卫星数据解压后, 哪个文件是业务元数据? :return: """ return CFile.join_file( self.file_content.content_root_dir, self.get_fuzzy_metadata_file( 'GF701.*_.*_.*_[0-9]{10}.xml', '{0}.xml'.format(self.classified_object_name())))
def get_transform_file_to_metadata_view(self, parser: CMetaDataParser): """ 对于需要由图像文件转换为预览图文件的方式进行处理 """ return CFile.join_file( self.get_sat_file_originally_path(), self.get_fuzzy_metadata_file( r'(?i).*_browse[.]tif$', '{0}_browse.tif'.format(self.classified_object_name()) ) )
def classified(self): """ 设计国土行业数据的dem_10_dem验证规则 完成 负责人 李宪 在这里检验dem_10_dem的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext check_file_main_name_length = len(file_main_name) == 13 if not check_file_main_name_length: return self.Object_Confirm_IUnKnown, self._object_name file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name) check_file_main_name_exist_tif = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) check_file_main_name_exist_bil = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Bil)) if (not check_file_main_name_exist_tif) and ( not check_file_main_name_exist_bil): return self.Object_Confirm_IUnKnown, self._object_name """ 下面判别第1位是字母 下面判别第4位是字母 下面判别第23位是数字 下面判别第567位是数字 下面判别第8910位是数字 下面判别第111213位是DOM """ char_1 = file_main_name[0:1] char_2_3 = file_main_name[1:3] char_4 = file_main_name[3:4] char_5_to_7 = file_main_name[4:7] char_8_to_10 = file_main_name[7:10] char_11_to_13 = file_main_name[10:13] if CUtils.text_is_alpha(char_1) is False \ or CUtils.text_is_numeric(char_2_3) is False \ or CUtils.text_is_alpha(char_4) is False \ or CUtils.text_is_numeric(char_5_to_7) is False \ or CUtils.text_is_numeric(char_8_to_10) is False \ or CUtils.equal_ignore_case(char_11_to_13, "DEM") is False: return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \ or CUtils.equal_ignore_case(file_ext, self.Name_Bil): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 王学谦 :param parser: :return: """ super().qa_file_custom(parser) file_object_name_list = re.findall(r'(?i)^([a-z]{2}\S+\d{4}[01]\d[0123]\d)[a-z][-]\d+$', self.file_info.file_main_name) file_object_name = file_object_name_list[0] # 去除尾部的F/M/P-数字 metadata_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) check_file_metadata_bus_exist = False ext = self.Transformer_XML temp_metadata_bus_file_Y = '{0}Y.xml'.format(metadata_main_name_with_path) temp_metadata_bus_file_P = '{0}P.xml'.format(metadata_main_name_with_path) temp_metadata_bus_file_M = '{0}M.xml'.format(metadata_main_name_with_path) if CFile.file_or_path_exist(temp_metadata_bus_file_Y): # 存在Y-P-M的优先级 check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = temp_metadata_bus_file_Y elif CFile.file_or_path_exist(temp_metadata_bus_file_P): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = temp_metadata_bus_file_P elif CFile.file_or_path_exist(temp_metadata_bus_file_M): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = temp_metadata_bus_file_M if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' } ) else: parser.metadata.quality.append_total_quality( { self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path) } )
def classified(self): """ 设计国土行业数据mosaic的验证规则(镶嵌影像) 完成 负责人 王学谦 在这里检验mosaic的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 初始化需要的参数 file_name_with_full_path = self.file_info.file_name_with_full_path file_object_name = file_main_name[:] # 主要名称截取 if file_name_with_full_path.endswith('_21at.xml'): # 元数据文件的情况 if len(file_main_name) > 5: file_object_name = file_main_name[:-5] else: return self.Object_Confirm_IUnKnown, self._object_name else: # 矢量文件的情况 xq_list = [ 'xq.shp', 'xq.shx', 'xq.dbf', 'xq.sbx', 'xq.prj', 'xq.sbn' ] for xq_end in xq_list: if file_name_with_full_path.lower().endswith(xq_end): if len(file_main_name) > 2: file_object_name = file_main_name[:-2] break else: return self.Object_Confirm_IUnKnown, self._object_name file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) check_file_main_name_exist = \ CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) or \ CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Img)) if not check_file_main_name_exist: # 检查主文件存在性 return self.Object_Confirm_IUnKnown, self._object_name check_file_shp_exist = CFile.file_or_path_exist( '{0}xq.shp'.format(file_main_name_with_path)) if not check_file_shp_exist: # 检查矢量文件存在性 return self.Object_Confirm_IUnKnown, self._object_name # 检查后缀名 if CUtils.equal_ignore_case(file_ext, self.Name_Tif) or CUtils.equal_ignore_case( file_ext, self.Name_Img): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name self.add_file_to_detail_list() else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def init_before_test(self): plugins_info = self.create_plugins().get_information() plugins_catalog = CUtils.dict_value_by_name( plugins_info, CPlugins.Plugins_Info_Catalog_Title, '') plugins_type = CUtils.dict_value_by_name(plugins_info, CPlugins.Plugins_Info_Type, '') self._test_file_root_path = settings.application.xpath_one( self.Path_Setting_Dir_Test_Data, '') self._test_file_parent_path = CFile.join_file( settings.application.xpath_one(self.Path_Setting_Dir_Test_Data, ''), plugins_catalog)
def plugins(cls, file_info: CDMFilePathInfoEx, plugins_id: str) -> CPlugins: """ 根据给定的文件信息和插件名称, 直接创建插件对象 如果插件文件不存在, 则返回None :param file_info: :param plugins_id: :return: """ target_type = file_info.file_type target_path = CFile.join_file(CSys.get_plugins_root_dir(), target_type) plugins_file_name = CFile.join_file( target_path, '{0}.{1}'.format(plugins_id, cls.FileExt_Py)) if not CFile.file_or_path_exist(plugins_file_name): return None plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), target_type) class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, plugins_id, file_info) return class_classified_obj
def init_qa_file_list(self, parser: CMetaDataParser) -> list: """ 设定镶嵌影像的质检列表,调用默认的主文件质检列表方法,并拼接剩余附属文件 完成 负责人 王学谦 在这里检验设定镶嵌影像的质检列表 """ file_main_name = self.classified_object_name() list_qa = list() list_qa.extend( self.init_qa_file_integrity_default_list( self.file_info.file_name_with_full_path)) # 调用默认的规则列表 # 获取xq.shp数据并判断完整性 xq_shp_with_full_path = CFile.join_file( self.file_info.file_path, '{0}xq.shp'.format(file_main_name)) list_qa.extend( self.init_qa_file_integrity_default_list(xq_shp_with_full_path)) # list_qa.extend([ # { # self.Name_FileName: '{0}xq.shp'.format(file_main_name), # self.Name_ID: 'shp', # self.Name_Title: 'shp文件', # self.Name_Group: self.QA_Group_Data_Integrity, # self.Name_Result: self.QA_Result_Error, # self.Name_Format: self.DataFormat_Vector_File # }, { # self.Name_FileName: '{0}xq.shx'.format(file_main_name), # self.Name_ID: 'shx', # self.Name_Title: 'shx文件', # self.Name_Group: self.QA_Group_Data_Integrity, # self.Name_Result: self.QA_Result_Error # }, { # self.Name_FileName: '{0}xq.dbf'.format(file_main_name), # self.Name_ID: 'dbf', # self.Name_Title: 'dbf文件', # self.Name_Group: self.QA_Group_Data_Integrity, # self.Name_Result: self.QA_Result_Error # }, { # self.Name_FileName: '{0}xq.sbx'.format(file_main_name), # self.Name_ID: 'sbx', # self.Name_Title: 'sbx文件', # self.Name_Group: self.QA_Group_Data_Integrity, # self.Name_Result: self.QA_Result_Warn # }, { # self.Name_FileName: '{0}xq.prj'.format(file_main_name), # self.Name_ID: 'prj', # self.Name_Title: 'prj文件', # self.Name_Group: self.QA_Group_Data_Integrity, # self.Name_Result: self.QA_Result_Warn # } # ]) return list_qa
def get_custom_affiliated_file_character(self): file_path = self.file_info.file_path letter_location = file_path.find('镶嵌影像成果') shp_path = CFile.join_file(file_path[:letter_location - 1], '影像时相接边图', self.get_yuji()) shp_regularexpression = '(?i)^.*_.*_' + self.get_coordinate_system() + r'\.shp$' return [ { self.Name_FilePath: shp_path, # 附属文件的路径 self.Name_RegularExpression: shp_regularexpression, # 附属文件的匹配规则 # 应该从上面匹配到的文件剔除的文件的匹配规则 self.Name_No_Match_RegularExpression: None } ]
def process(self) -> str: """ 注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!! :return: """ result_process = self.process_vector() if CResult.result_success(result_process): file_path = self.file_content.work_root_dir dict_temp_file_name = { self.Name_Native_Center: '{0}_native_center.wkt'.format(self.object_name), self.Name_Native_BBox: '{0}_native_bbox.wkt'.format(self.object_name), self.Name_Native_Geom: '{0}_native_geom.wkt'.format(self.object_name), self.Name_Wgs84_Center: '{0}_wgs84_center.wkt'.format(self.object_name), self.Name_Wgs84_BBox: '{0}_wgs84_bbox.wkt'.format(self.object_name), self.Name_Wgs84_Geom: '{0}_wgs84_geom.wkt'.format(self.object_name) } dict_temp_prj_name = { self.Name_Prj_Wkt: CResult.result_info(result_process, self.Name_Prj_Wkt, None), self.Name_Prj_Proj4: CResult.result_info(result_process, self.Name_Prj_Proj4, None), self.Name_Prj_Project: CResult.result_info(result_process, self.Name_Prj_Project, None), self.Name_Prj_Coordinate: CResult.result_info(result_process, self.Name_Prj_Coordinate, None), self.Name_Prj_Source: CResult.result_info(result_process, self.Name_Prj_Source, None), self.Name_Prj_Zone: CResult.result_info(result_process, self.Name_Prj_Zone, None), self.Name_Prj_Degree: CResult.result_info(result_process, self.Name_Prj_Degree, None) } result = CResult.merge_result(self.Success, '处理完毕!') for file_type, file_name in dict_temp_file_name.items(): result = CResult.merge_result_info( result, file_type, CFile.join_file(file_path, file_name)) for prj_type, prj_name in dict_temp_prj_name.items(): result = CResult.merge_result_info(result, prj_type, prj_name) else: result = CResult.merge_result( self.Failure, CResult.result_message(result_process)) return result
def classified(self): """ 设计国土行业数据的dom-10验证规则 完成 负责人 李宪 在这里检验dem-10的元数据文件格式时, 应该一个一个类型的对比, 找到文件时, 将该文件的格式和文件名存储到类的私有属性中, 以便在元数据处理时直接使用 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext check_file_main_name_length = len(file_main_name) == 10 if not check_file_main_name_length: return self.Object_Confirm_IUnKnown, self._object_name file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name) check_file_main_name_exist_tif = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) check_file_main_name_exist_bil = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Bil)) if (not check_file_main_name_exist_tif) and ( not check_file_main_name_exist_bil): return self.Object_Confirm_IUnKnown, self._object_name """ 下面判别第1位是字母 下面判别第4位是字母 下面判别第23位是数字 下面判别第567位是数字 下面判别第8910位是数字 """ char_1 = file_main_name[0:1] char_2_3 = file_main_name[1:3] char_4 = file_main_name[3:4] char_5_to_7 = file_main_name[4:7] char_8_to_10 = file_main_name[7:10] if CUtils.text_is_alpha(char_1) is False \ or CUtils.text_is_numeric(char_2_3) is False \ or CUtils.text_is_alpha(char_4) is False \ or CUtils.text_is_numeric(char_5_to_7) is False \ or CUtils.text_is_numeric(char_8_to_10) is False: return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \ or CUtils.equal_ignore_case(file_ext, self.Name_Bil): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def __plugins_classified_of_directory__( cls, file_info: CDMFilePathInfoEx) -> CPlugins: """ 使用系统目录下的所有插件进行识别 :param file_info: :return: """ target = file_info.file_main_name target_type = file_info.file_type plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), target_type) path = CFile.join_file(CSys.get_plugins_root_dir(), target_type) plugins_file_list = CFile.file_or_subpath_of_path( path, '{0}_*.{1}'.format(cls.Name_Plugins, cls.FileExt_Py)) for file_name_without_path in plugins_file_list: file_main_name = CFile.file_main_name(file_name_without_path) try: class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, file_main_name, file_info) obj_info = class_classified_obj.get_information() # 王西亚 添加逻辑于2021-02-12, 解决项目插件调试不方便的问题 obj_classified_valid = False obj_owner_project_id = CUtils.dict_value_by_name( obj_info, CPlugins.Plugins_Info_Project_ID, None) system_project_id = settings.application.xpath_one( cls.Path_Setting_Project_ID, None) if obj_owner_project_id is None: obj_classified_valid = True else: obj_classified_valid = CUtils.equal_ignore_case( obj_owner_project_id, system_project_id) if obj_classified_valid: object_confirm, object_name = class_classified_obj.classified( ) if object_confirm != cls.Object_Confirm_IUnKnown: obj_id = class_classified_obj.get_id() CLogger().debug('{0} is classified as {1}.{2}'.format( target, obj_info, obj_id)) return class_classified_obj except Exception as error: CLogger().debug('插件[{0}]解析出现异常, 错误信息为: [{1}], 请检查!'.format( file_main_name, error.__str__())) if settings.application.xpath_one( '{0}.{1}'.format(cls.Name_Application, cls.Name_Debug), cls.DB_False) == cls.DB_True: raise else: continue else: return None
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 李宪 :param parser: :return: """ super().qa_file_custom(parser) file_metadata_name_with_path = CFile.join_file( self.file_info.file_path, self.file_info.file_main_name) check_file_metadata_name_exist = False ext_list = ['xls', 'xlsx', 'mat', 'mdb'] for ext in ext_list: temp_metadata_bus_file = '{0}.{1}'.format( file_metadata_name_with_path, ext) if CFile.file_or_path_exist(temp_metadata_bus_file): check_file_metadata_name_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = temp_metadata_bus_file break if not check_file_metadata_name_exist: parser.metadata.quality.append_total_quality({ self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' }) else: parser.metadata.quality.append_total_quality({ self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path) })
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 王学谦 :param parser: :return: """ super().qa_file_custom(parser) file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name check_file_metadata_bus_exist = False ext = self.Transformer_XML metadata_name_with_path = CFile.join_file( file_path, '{0}M.xml'.format(file_main_name[:-1])) if CFile.file_or_path_exist(metadata_name_with_path): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = metadata_name_with_path if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality({ self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' }) else: parser.metadata.quality.append_total_quality({ self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format( CFile.file_name(self.metadata_bus_src_filename_with_path)) })
def process(self) -> str: """ 完成 负责人 赵宇飞 在这里提取影像数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式 注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!! :return: """ out_metadata_file_fullname = CFile.join_file(self.file_content.work_root_dir, self.FileName_MetaData) raster_mdreader = CRasterMDReader(self.file_info.file_name_with_full_path) # result = raster_mdreader.get_metadata_2_file(out_metadata_file_fullname) result = CProcessUtils.processing_method(raster_mdreader.get_metadata_2_file, out_metadata_file_fullname) # 进程调用模式 # p_one = Process(target=raster_mdreader.get_metadata_2_file, args=(out_metadata_file_fullname,)) # p_one.start() # p_one.join() return CResult.merge_result_info(result, self.Name_Format, self.MetaDataFormat_Json)
def process_mission(self, dataset) -> str: """ :param dataset: :return: """ ds_na_id = dataset.value_by_name(0, 'na_id', '') ds_app_id = dataset.value_by_name(0, 'app_id', '') ds_object_id = dataset.value_by_name(0, 'object_id', '') ds_object_type = dataset.value_by_name(0, 'object_type', '') ds_object_name = dataset.value_by_name(0, 'object_name', '') ds_object_access = dataset.value_by_name(0, 'object_access', self.DataAccess_Forbid) CLogger().debug('与第三方模块[{0}]同步的对象为: [{1}]'.format( ds_app_id, ds_object_name)) try: module_file_name = CFile.join_file( CSys.get_metadata_data_access_modules_root_dir(), '{0}.{1}'.format(ds_app_id, self.FileExt_Py)) if not CFile.file_or_path_exist(module_file_name): message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id) result = CResult.merge_result(self.Success, message) self.update_sync_result(ds_na_id, result) return result module_obj = CObject.create_module_instance( CSys.get_metadata_data_access_modules_root_name(), ds_app_id, self.get_mission_db_id()) if module_obj is None: message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id) result = CResult.merge_result(self.Success, message) self.update_sync_result(ds_na_id, result) return result module_title = CUtils.dict_value_by_name(module_obj.information(), self.Name_Title, '') result = module_obj.sync(ds_object_access, ds_object_id, ds_object_name, ds_object_type, None) self.update_sync_result(ds_na_id, result) return result except Exception as error: result = CResult.merge_result( self.Failure, '与第三方模块[{0}]同步的对象: [{1}]的同步过程出现异常, 详细情况: [{2}]!'.format( ds_app_id, ds_object_name, error.__str__())) self.update_sync_result(ds_na_id, result) return result
def process(self) -> str: """ 在这里提取矢量数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式 :return: """ result = super().process() metadata_main_name_with_path = CFile.join_file( self.file_info.file_path, self.file_info.file_main_name) metadata_filename = '{0}.xml'.format(metadata_main_name_with_path[:-2]) format_metadata = self.MetaDataFormat_XML result = CResult.merge_result_info(result, self.Name_FileName, metadata_filename) return CResult.merge_result_info(result, self.Name_Format, format_metadata)
def classified(self): """ 设计国土行业数据的dom-12验证规则 完成 负责人 李宪 在这里检验dom-12的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext check_file_main_name_length = len(file_main_name) == 12 if not check_file_main_name_length: return self.Object_Confirm_IUnKnown, self._object_name file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name) check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) if not check_file_main_name_exist: return self.Object_Confirm_IUnKnown, self._object_name """ 下面判别第1位是字母 下面判别第4位是字母 下面判别第23位是数字 下面判别第567位是数字 下面判别第8910位是数字 """ char_1 = file_main_name[0:1] char_2_3 = file_main_name[1:3] char_4 = file_main_name[3:4] char_5_to_7 = file_main_name[4:7] char_8_to_12 = file_main_name[7:12] if CUtils.text_is_alpha(char_1) is False \ or CUtils.text_is_numeric(char_2_3) is False \ or CUtils.text_is_alpha(char_4) is False \ or CUtils.text_is_numeric(char_5_to_7) is False \ or CUtils.text_is_numeric(char_8_to_12) is False: return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 王学谦 :param parser: :return: """ super().qa_file_custom(parser) metadata_main_name_with_path = CFile.join_file( self.file_info.file_path, self.file_info.file_main_name) check_file_metadata_bus_exist = False temp_metadata_bus_file = '{0}.xml'.format( metadata_main_name_with_path[:-2]) if CFile.file_or_path_exist(temp_metadata_bus_file): check_file_metadata_bus_exist = True if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality({ self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' }) else: parser.metadata.quality.append_total_quality({ self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path) })
def is_raster_file_integrity(cls, raster_file_with_path: str) -> bool: """ 判断影像数据的文件完整性,img xxx.img 文件 栅格数据可读 错误 xxx.ige 文件 img小于1M时必须存在 警告 @param raster_file_with_path: @return: """ file_ext = CFile.file_ext(raster_file_with_path) if file_ext.lower() == 'img': size = CFile.file_size(raster_file_with_path) if size < 1024 * 1024: file_main_name = CFile.file_main_name(raster_file_with_path) file_path = CFile.file_path(raster_file_with_path) ige = CFile.join_file(file_path, file_main_name + '.ige') if not CFile.file_or_path_exist(ige): return False return True
def classified(self): file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 初始化需要的参数 file_object_name = file_main_name[:] file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) if CUtils.equal_ignore_case(file_ext, self.Name_Shp): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: if CFile.file_or_path_exist('{0}.{1}'.format( file_main_name_with_path, self.Name_Shp)): self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None else: self._object_confirm = self.Object_Confirm_IUnKnown self._object_name = None return self._object_confirm, self._object_name
def get_custom_affiliated_file_character(self): file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name regularexpression = '(?i)^' + file_main_name[:-1] + r'.\..*' letter_location = file_path.find('FenFu') shp_path = CFile.join_file(file_path[:letter_location - 1], 'FenFu', self.get_coordinate_system_title()) return [ { self.Name_FilePath: file_path, # 附属文件的路径 self.Name_RegularExpression: regularexpression, # 附属文件的匹配规则 # 应该从上面匹配到的文件剔除的文件的匹配规则 self.Name_No_Match_RegularExpression: '(?i)^' + file_main_name + r'\..*$' }, { self.Name_FilePath: shp_path, # 附属文件的路径 self.Name_RegularExpression: r'(?i)\.shp$', # 附属文件的匹配规则 # 应该从上面匹配到的文件剔除的文件的匹配规则 self.Name_No_Match_RegularExpression: None } ]
def db_update_status_on_path_invalid(self): """ 处理目录不存在时的业务 1. 标记已经入库的文件为无效 1. 标记已经入库的子目录为无效 :return: """ path_name_with_relation_path = CFile.join_file( self.file_path_with_rel_path, '') params = dict() params['dsdStorageID'] = self.storage_id params['dsdSubDirectory'] = path_name_with_relation_path sql_update_file_invalid = ''' update dm2_storage_file set dsffilevalid = 0, dsfscanstatus = 0 where dsfdirectoryid in ( select dsdid from dm2_storage_directory where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1 ) '''.format(CFile.sep()) sql_update_path_invalid = ''' update dm2_storage_directory set dsd_directory_valid = 0, dsdscanstatus = 0, dsdscanfilestatus = 0, dsdscandirstatus = 0 where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1 '''.format(CFile.sep()) engine = CFactory().give_me_db(self.db_server_id) session = engine.give_me_session() try: engine.session_execute(session, sql_update_file_invalid, params) engine.session_execute(session, sql_update_path_invalid, params) engine.session_commit(session) except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__())) engine.session_rollback(session) finally: engine.session_close(session)
def get_custom_affiliated_file_character(self): file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name regularexpression = '(?i)^' + file_main_name[:-1] + r'.\..*' letter_location_FenFu = file_path.find('FenFu') file_path_sub = file_path[letter_location_FenFu + 6:] letter_location = file_path_sub.find(CFile.sep()) shp_path = CFile.join_file(file_path[:letter_location_FenFu + 5], file_path_sub[:letter_location]) return [ { self.Name_FilePath: file_path, # 附属文件的路径 self.Name_RegularExpression: regularexpression, # 附属文件的匹配规则 # 应该从上面匹配到的文件剔除的文件的匹配规则 self.Name_No_Match_RegularExpression: '(?i)^' + file_main_name + r'\..*$' }, { self.Name_FilePath: shp_path, # 附属文件的路径 self.Name_RegularExpression: r'(?i)\.shp$', # 附属文件的匹配规则 # 应该从上面匹配到的文件剔除的文件的匹配规则 self.Name_No_Match_RegularExpression: None } ]
def test_metadata_time(self): self.init_before_test() # 初始化路径 for test_file_info in self.test_file_info_list(): file_type, test_file_with_rel_path, correct_object_confirm, correct_object_name = \ self.get_test_file_info(test_file_info) if correct_object_confirm == self.Object_Confirm_IKnown: test_file_with_full_path = CFile.join_file(self._test_file_parent_path, test_file_with_rel_path) # 获取插件对象 file_info, plugins_obj, metadata_parser = self.get_test_obj(file_type, test_file_with_full_path) # 执行测试 plugins_obj.parser_metadata_with_qa(metadata_parser) plugins_obj.parser_metadata_time_after_qa(metadata_parser) # 获取结果 result_with_time, message_with_time, time_information = metadata_parser.metadata.metadata_time() plugins_obj.destroy_virtual_content() # 录入测试信息 allure.attach('处理信息为{0},时间元数据内容为{1}' .format(message_with_time, time_information), '{0}'.format(test_file_with_rel_path), allure.attachment_type.TEXT) assert result_with_time
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 李宪 :param parser: :return: """ metadata_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name) check_file_metadata_bus_exist = False ext = self.Transformer_XML temp_metadata_bus_file = '{0}_21at.xml'.format(metadata_main_name_with_path) if CFile.file_or_path_exist(temp_metadata_bus_file): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = temp_metadata_bus_file if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据,是正常现象' } ) else: parser.metadata.quality.append_total_quality( { self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path) } )