def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 王学谦 :param parser: :return: """ super().qa_file_custom(parser) file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name check_file_metadata_bus_exist = False ext = self.Transformer_XML metadata_name_with_path = CFile.join_file( file_path, '{0}M.xml'.format(file_main_name[:-1])) if CFile.file_or_path_exist(metadata_name_with_path): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = metadata_name_with_path if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality({ self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' }) else: parser.metadata.quality.append_total_quality({ self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format( CFile.file_name(self.metadata_bus_src_filename_with_path)) })
def classified(self): """ 设计国土行业数据的dom_10_dom验证规则 完成 负责人 李宪 在这里检验dom_10_dom的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext check_file_main_name_length = len(file_main_name) == 13 if not check_file_main_name_length: return self.Object_Confirm_IUnKnown, self._object_name file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name) check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) if not check_file_main_name_exist: return self.Object_Confirm_IUnKnown, self._object_name """ 下面判别第1位是字母 下面判别第4位是字母 下面判别第23位是数字 下面判别第567位是数字 下面判别第8910位是数字 下面判别第111213位是DOM """ char_1 = file_main_name[0:1] char_2_3 = file_main_name[1:3] char_4 = file_main_name[3:4] char_5_to_7 = file_main_name[4:7] char_8_to_10 = file_main_name[7:10] char_11_to_13 = file_main_name[10:13] if CUtils.text_is_alpha(char_1) is False \ or CUtils.text_is_numeric(char_2_3) is False \ or CUtils.text_is_alpha(char_4) is False \ or CUtils.text_is_numeric(char_5_to_7) is False \ or CUtils.text_is_numeric(char_8_to_10) is False \ or CUtils.equal_ignore_case(char_11_to_13, "DOM") is False: return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def file2param(cls, params: dict, param_name: str, file_name: str): if params is None: return if not CFile.file_or_path_exist(file_name): params[param_name] = None return # 注意这里一定要使用rb,读出二进制文件,否则有读不全等问题 fp = open(file_name, 'rb') try: params[param_name] = fp.read() finally: fp.close()
def classified(self): """ 设计国土行业数据custom的验证规则(自定义影像) 完成 负责人 李宪 在这里检验custom的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext file_object_name = file_main_name file_name_with_full_path = self.file_info.file_name_with_full_path # 初始化需要的参数 if file_name_with_full_path.endswith('_21at.xml'): file_object_name = file_main_name[:-5] file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) check_file_main_name_exist_tif = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) check_file_main_name_exist_img = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Img)) if (not check_file_main_name_exist_tif) and ( not check_file_main_name_exist_img): return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \ or CUtils.equal_ignore_case(file_ext, self.Name_Img): self._object_confirm = self.Object_Confirm_IKnown self._object_name = self.file_info.file_main_name file_detail_xml = '{0}_21at.xml'.format( self.file_info.file_main_name_with_full_path) self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中 else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def process_mission(self, dataset) -> str: """ :param dataset: :return: """ ds_na_id = dataset.value_by_name(0, 'na_id', '') ds_app_id = dataset.value_by_name(0, 'app_id', '') ds_object_id = dataset.value_by_name(0, 'object_id', '') ds_object_type = dataset.value_by_name(0, 'object_type', '') ds_object_name = dataset.value_by_name(0, 'object_name', '') ds_object_access = dataset.value_by_name(0, 'object_access', self.DataAccess_Forbid) CLogger().debug('与第三方模块[{0}]同步的对象为: [{1}]'.format( ds_app_id, ds_object_name)) try: module_file_name = CFile.join_file( CSys.get_metadata_data_access_modules_root_dir(), '{0}.{1}'.format(ds_app_id, self.FileExt_Py)) if not CFile.file_or_path_exist(module_file_name): message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id) result = CResult.merge_result(self.Success, message) self.update_sync_result(ds_na_id, result) return result module_obj = CObject.create_module_instance( CSys.get_metadata_data_access_modules_root_name(), ds_app_id, self.get_mission_db_id()) if module_obj is None: message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id) result = CResult.merge_result(self.Success, message) self.update_sync_result(ds_na_id, result) return result module_title = CUtils.dict_value_by_name(module_obj.information(), self.Name_Title, '') result = module_obj.sync(ds_object_access, ds_object_id, ds_object_name, ds_object_type, None) self.update_sync_result(ds_na_id, result) return result except Exception as error: result = CResult.merge_result( self.Failure, '与第三方模块[{0}]同步的对象: [{1}]的同步过程出现异常, 详细情况: [{2}]!'.format( ds_app_id, ds_object_name, error.__str__())) self.update_sync_result(ds_na_id, result) return result
def classified(self): """ 设计国土行业数据的dom-10验证规则 完成 负责人 李宪 在这里检验dom-10的元数据文件格式时, 应该一个一个类型的对比, 找到文件时, 将该文件的格式和文件名存储到类的私有属性中, 以便在元数据处理时直接使用 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext check_file_main_name_length = len(file_main_name) == 10 if not check_file_main_name_length: return self.Object_Confirm_IUnKnown, self._object_name file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name) check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format( file_main_name_with_path, self.Name_Tif)) if not check_file_main_name_exist: return self.Object_Confirm_IUnKnown, self._object_name """ 下面判别第1位是字母 下面判别第4位是字母 下面判别第23位是数字 下面判别第567位是数字 下面判别第8910位是数字 """ char_1 = file_main_name[0:1] char_2_3 = file_main_name[1:3] char_4 = file_main_name[3:4] char_5_to_7 = file_main_name[4:7] char_8_to_10 = file_main_name[7:10] if CUtils.text_is_alpha(char_1) is False \ or CUtils.text_is_numeric(char_2_3) is False \ or CUtils.text_is_alpha(char_4) is False \ or CUtils.text_is_numeric(char_5_to_7) is False \ or CUtils.text_is_numeric(char_8_to_10) is False: return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def create_virtual_content(self) -> bool: """ :return: """ zip_obj = CZip(self.__target_name__) try: zip_obj.open() zip_obj.extract_all(self.__virtual_content_root_dir__) except: return False finally: zip_obj.close() return super().create_virtual_content() and CFile.file_or_path_exist( self.__virtual_content_root_dir__)
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 王学谦 :param parser: :return: """ super().qa_file_custom(parser) metadata_main_name_with_path = CFile.join_file( self.file_info.file_path, self.file_info.file_main_name) check_file_metadata_bus_exist = False temp_metadata_bus_file = '{0}.xml'.format( metadata_main_name_with_path[:-2]) if CFile.file_or_path_exist(temp_metadata_bus_file): check_file_metadata_bus_exist = True if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality({ self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' }) else: parser.metadata.quality.append_total_quality({ self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path) })
def is_raster_file_integrity(cls, raster_file_with_path: str) -> bool: """ 判断影像数据的文件完整性,img xxx.img 文件 栅格数据可读 错误 xxx.ige 文件 img小于1M时必须存在 警告 @param raster_file_with_path: @return: """ file_ext = CFile.file_ext(raster_file_with_path) if file_ext.lower() == 'img': size = CFile.file_size(raster_file_with_path) if size < 1024 * 1024: file_main_name = CFile.file_main_name(raster_file_with_path) file_path = CFile.file_path(raster_file_with_path) ige = CFile.join_file(file_path, file_main_name + '.ige') if not CFile.file_or_path_exist(ige): return False return True
def classified(self): file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 初始化需要的参数 file_object_name = file_main_name[:] file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) if CUtils.equal_ignore_case(file_ext, self.Name_Shp): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: if CFile.file_or_path_exist('{0}.{1}'.format( file_main_name_with_path, self.Name_Shp)): self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None else: self._object_confirm = self.Object_Confirm_IUnKnown self._object_name = None return self._object_confirm, self._object_name
def plugins(cls, file_info: CDMFilePathInfoEx, plugins_id: str) -> CPlugins: """ 根据给定的文件信息和插件名称, 直接创建插件对象 如果插件文件不存在, 则返回None :param file_info: :param plugins_id: :return: """ target_type = file_info.file_type target_path = CFile.join_file(CSys.get_plugins_root_dir(), target_type) plugins_file_name = CFile.join_file( target_path, '{0}.{1}'.format(plugins_id, cls.FileExt_Py)) if not CFile.file_or_path_exist(plugins_file_name): return None plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), target_type) class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, plugins_id, file_info) return class_classified_obj
def __process_file_format__(self, metadata_format: str): if not CFile.file_or_path_exist(self.transformer_src_filename): return CResult.merge_result( self.Failure, '文件[{0}]不存在, 无法解析! '.format(self.transformer_src_filename)) try: self.__metadata__.set_metadata_bus_file( self.Success, '文件[{0}]成功加载! '.format(self.transformer_src_filename), metadata_format, self.transformer_src_filename) return CResult.merge_result( self.Success, '文件[{0}]成功加载! '.format(self.transformer_src_filename)) except: self.__metadata__.set_metadata_bus( self.Exception, '元数据文件[{0}]格式不合法, 无法处理! '.format( self.transformer_src_filename), self.MetaDataFormat_Text, '') return CResult.merge_result( self.Exception, '元数据文件[{0}]格式不合法, 无法处理! '.format( self.transformer_src_filename))
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 李宪 :param parser: :return: """ metadata_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name) check_file_metadata_bus_exist = False ext = self.Transformer_XML temp_metadata_bus_file = '{0}_21at.xml'.format(metadata_main_name_with_path) if CFile.file_or_path_exist(temp_metadata_bus_file): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = temp_metadata_bus_file if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据,是正常现象' } ) else: parser.metadata.quality.append_total_quality( { self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path) } )
def open(self, encoding: str = 'UTF-8'): gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES") # gdal.SetConfigOption("SHAPE_ENCODING", "GBK") gdal.SetConfigOption("SHAPE_ENCODING", encoding) # 注册所有的驱动 ogr.RegisterAll() self._active = False if not CFile.file_or_path_exist(self.data_file_or_path): raise FileNotFoundError(self.data_file_or_path) self._data_obj = ogr.Open(self.data_file_or_path, 0) mdb_flag = False if self._data_obj is None: self._data_obj = ogr.Open( self.data_file_or_path.encode(CResource.Encoding_GBK), 0) mdb_flag = True if self._data_obj is None: raise FileCanNotOpenException(self.data_file_or_path) self._active = True self._parser_datasets(mdb_flag)
def process_mission(self, dataset) -> str: """ 详细算法复杂, 参见readme.md中[### 数据入库调度]章节 :param dataset: :return: """ ds_src_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_src_storage_type = dataset.value_by_name(0, 'query_storage_type', self.Storage_Type_Mix) ds_src_root_path = dataset.value_by_name(0, 'query_rootpath', '') ds_src_dir_id = dataset.value_by_name(0, 'query_ib_dir_id', '') ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '') ds_ib_directory_name = dataset.value_by_name(0, 'query_ib_relation_dir', '') ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '') ds_ib_option = dataset.value_by_name(0, 'query_ib_option', '') src_need_storage_size = self.get_storage_size(ds_ib_id, ds_src_storage_id, ds_ib_directory_name, ds_ib_option) src_path = ds_src_root_path if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): src_path = CFile.join_file(src_path, ds_ib_directory_name) src_dataset_metadata_filename = CFile.join_file( src_path, self.FileName_MetaData_Bus_21AT) CLogger().debug('入库的目录为: {0}.{1}'.format(ds_ib_id, ds_ib_directory_name)) try: # 检查所有文件与元数据是否相符 all_ib_file_or_path_existed = self.check_all_ib_file_or_path_existed( ds_ib_id) if not CResult.result_success(all_ib_file_or_path_existed): self.update_ib_result(ds_ib_id, all_ib_file_or_path_existed) return all_ib_file_or_path_existed # 将数据入库的记录保存到日志中 result = self.ib_log(ds_ib_id, ds_src_storage_id, ds_ib_directory_name) if not CResult.result_success(result): self.update_ib_result(ds_ib_id, result) return result # 如果是在核心存储或混合存储中直接入库, 则仅仅改变元数据状态即可 if CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Mix) \ or CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Core): result_ib_in_core_or_mix_storage = self.update_ib_data_status_in_core_or_mix_storage( ds_ib_id, ds_src_storage_id, ds_ib_directory_name, ds_src_dir_id) self.update_ib_result(ds_ib_id, result_ib_in_core_or_mix_storage) return result_ib_in_core_or_mix_storage # 加载目录下的待入库数据集的元数据文件 src_dataset_xml = CXml() src_dataset_type = self.Name_Default if CFile.file_or_path_exist(src_dataset_metadata_filename): src_dataset_xml.load_file(src_dataset_metadata_filename) src_dataset_type = CXml.get_element_text( src_dataset_xml.xpath_one(self.Path_MD_Bus_ProductType)) if CUtils.equal_ignore_case(src_dataset_type, ''): src_dataset_type = self.Name_Default # 获取匹配的入库模式 src_ib_schema = self.get_ib_schema(src_dataset_type, ds_ib_option) if src_ib_schema is None: result = CResult.merge_result( self.Failure, '目录为[{0}.{1}]的数据集类型为[{2}], 未找到匹配的入库模式, 请检查修正后重试!'.format( ds_ib_id, ds_ib_directory_name, src_dataset_type)) self.update_ib_result(ds_ib_id, result) return result # 计算入库的目标存储\存储根目录\目标子目录在目标存储中的副目录的标识\目标子目录\反馈消息 dest_ib_storage_id, dest_ib_root_path, desc_ib_dir_id, dest_ib_subpath, message = self.get_dest_storage( ds_ib_batch_no, src_need_storage_size, ds_ib_option, src_ib_schema, src_dataset_xml) if dest_ib_storage_id is None or dest_ib_subpath is None: result = CResult.merge_result(self.Failure, message) self.update_ib_result(ds_ib_id, result) return result dest_ib_subpath = CFile.unify(dest_ib_subpath) if CJson.json_attr_value(ds_ib_option, self.Path_IB_Switch_CheckFileLocked, self.DB_False) == self.DB_True: src_ib_files_not_locked, message = self.check_src_ib_files_not_locked( ds_src_root_path, src_path) if not src_ib_files_not_locked: result = CResult.merge_result(self.Failure, message) self.update_ib_result(ds_ib_id, result) return result proc_ib_src_path = ds_src_root_path proc_ib_dest_path = dest_ib_root_path if not CUtils.equal_ignore_case(dest_ib_subpath, ''): proc_ib_dest_path = CFile.join_file(dest_ib_root_path, dest_ib_subpath) if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): proc_ib_src_path = CFile.join_file(proc_ib_src_path, ds_ib_directory_name) proc_ib_dest_path = CFile.join_file(proc_ib_dest_path, ds_ib_directory_name) # --------------------------------------------------------------至此, 数据入库前的检查处理完毕 # 移动源目录至目标目录, 如果是根目录, 则仅仅移动文件 result = self.ib_files_move( proc_ib_src_path, proc_ib_dest_path, CUtils.equal_ignore_case(ds_ib_directory_name, '')) if not CResult.result_success(result): # 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功 sub_result = self.ib_files_move( proc_ib_dest_path, proc_ib_src_path, CUtils.equal_ignore_case(ds_ib_directory_name, '')) if not CResult.result_success(sub_result): sub_result_message = CResult.result_message(sub_result) result_message = CResult.result_message(result) result = CResult.merge_result( self.Failure, '{0}\n{1}'.format(result_message, sub_result_message)) self.update_ib_result(ds_ib_id, result) return result # 将源文件的元数据, 移动至目标存储下, 如果出现异常, 则在方法内部rollback result = self.src_ib_metadata_move_to_storage( ds_ib_id, ds_src_storage_id, ds_src_dir_id, ds_ib_directory_name, dest_ib_storage_id, desc_ib_dir_id, dest_ib_subpath) if not CResult.result_success(result): # 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功 sub_result = self.ib_files_move( proc_ib_dest_path, proc_ib_src_path, CUtils.equal_ignore_case(ds_ib_directory_name, '')) if not CResult.result_success(sub_result): sub_result_message = CResult.result_message(sub_result) result_message = CResult.result_message(result) result = CResult.merge_result( self.Failure, '{0}/n{1}'.format(result_message, sub_result_message)) self.update_ib_result(ds_ib_id, result) return result result = CResult.merge_result( self.Success, '目录为[{0}.{1}]入库成功!'.format(ds_ib_id, ds_ib_directory_name)) self.update_ib_result(ds_ib_id, result) return result except Exception as error: result = CResult.merge_result( self.Failure, '目录为[{0}.{1}]入库出现异常! 错误原因为: {2}'.format( ds_ib_id, ds_ib_directory_name, error.__str__())) self.update_ib_result(ds_ib_id, result) return result
def process(self) -> str: """ 在这里提取文档数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式 注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!! :return: """ default_result = super().process() out_metadata_file_fullname = CFile.join_file( self.file_content.work_root_dir, self.FileName_MetaData) in_file_fullname = self.file_info.file_name_with_full_path if not settings.application.xpath_one( self.Path_Setting_Dependence_Tika_Enable, True): return default_result tika_dependence_mode = settings.application.xpath_one( self.Path_Setting_Dependence_Tika_Mode, self.Name_Server) if CUtils.equal_ignore_case(tika_dependence_mode, self.Name_Server): tika_server_url = settings.application.xpath_one( self.Path_Setting_Dependence_Tika_Server_Url, None) tika_server_connect_timeout = settings.application.xpath_one( self.Path_Setting_Dependence_Tika_Server_Timeout, 30) if CUtils.equal_ignore_case(tika_server_url, ''): return default_result try: parsed = TikaServer.from_file( in_file_fullname, tika_server_url, requestOptions={'timeout': tika_server_connect_timeout}) meta_data_dict = parsed["metadata"] json_obj = CJson() json_obj.load_obj(meta_data_dict) json_obj.to_file(out_metadata_file_fullname) return CResult.merge_result_info( CResult.merge_result( self.Success, '文档[{0}]的元数据提取成功'.format(in_file_fullname)), self.Name_Format, self.MetaDataFormat_Json) except Exception as error: return CResult.merge_result( self.Failure, '文档[{0}]的元数据提取过程出现错误, 详细信息为: [{1}]'.format( in_file_fullname, error.__str__())) else: tika_application = settings.application.xpath_one( self.Path_Setting_Dependence_Tika_Client_App, None) if CUtils.equal_ignore_case(tika_application, ''): return default_result if not CFile.file_or_path_exist(tika_application): return CResult.merge_result( self.Failure, '文档[{0}]的元数据无法提取, 详细原因为: [依赖中间件{1}文件不存在, 请修正后重试!]'.format( in_file_fullname, tika_application)) try: tika_client = TikaApplication(file_jar=tika_application) meta_data_dict = tika_client.extract_only_metadata( in_file_fullname) json_obj = CJson() json_obj.load_obj(meta_data_dict) json_obj.to_file(out_metadata_file_fullname) return CResult.merge_result_info( CResult.merge_result( self.Success, '文档[{0}]的元数据提取成功'.format(in_file_fullname)), self.Name_Format, self.MetaDataFormat_Json) except Exception as error: return CResult.merge_result( self.Failure, '文档[{0}]的元数据提取过程出现错误, 详细信息为: [{1}]'.format( in_file_fullname, error.__str__())) # result = raster_mdreader.get_metadata_2_file(out_metadata_file_fullname) # result = CProcessUtils.processing_method(raster_mdreader.get_metadata_2_file, out_metadata_file_fullname) # 进程调用模式 # p_one = Process(target=raster_mdreader.get_metadata_2_file, args=(out_metadata_file_fullname,)) # p_one.start() # p_one.join() return CResult.merge_result_info(result, self.Name_Format, self.MetaDataFormat_Json)
def save_metadata_spatial(self) -> str: """ 完成空间元数据的入库更新操作 :return: """ mdt_spatial_result, mdt_spatial_memo, mdt_spatial = self.metadata.metadata_spatial() params = { 'dsoid': self.object_id, 'dso_spatial_result': mdt_spatial_result, 'dso_spatial_parsermemo': mdt_spatial_memo, 'dso_prj_wkt': mdt_spatial.prj_wkt, 'dso_prj_proj4': mdt_spatial.prj_proj4, 'dso_prj_project': mdt_spatial.prj_project, 'dso_prj_coordinate': mdt_spatial.prj_coordinate, 'dso_prj_degree': mdt_spatial.prj_degree, 'dso_prj_zone': mdt_spatial.prj_zone, 'dso_prj_source': mdt_spatial.prj_source } database = CFactory().give_me_db(self.file_info.db_server_id) params['dso_center_native'] = CFile.file_2_str(mdt_spatial.native_center) params['dso_geo_bb_native'] = CFile.file_2_str(mdt_spatial.native_box) params['dso_geo_native'] = CFile.file_2_str(mdt_spatial.native_geom) if CFile.file_or_path_exist(mdt_spatial.wgs84_center): dso_center_wgs84 = database.sql.func_wkt2geometry( CUtils.quote(CFile.file_2_str(mdt_spatial.wgs84_center)), self.SRID_WGS84) else: dso_center_wgs84 = 'null' if CFile.file_or_path_exist(mdt_spatial.wgs84_bbox): dso_geo_bb_wgs84 = database.sql.func_wkt2geometry( CUtils.quote( CFile.file_2_str(mdt_spatial.wgs84_bbox)), self.SRID_WGS84) else: dso_geo_bb_wgs84 = 'null' if CFile.file_or_path_exist(mdt_spatial.wgs84_geom): dso_geo_wgs84 = database.sql.func_wkt2geometry( CUtils.quote( CFile.file_2_str(mdt_spatial.wgs84_geom)), self.SRID_WGS84) else: dso_geo_wgs84 = 'null' # 所有元数据入库 CFactory().give_me_db(self.file_info.db_server_id).execute( ''' update dm2_storage_object set dso_spatial_result = :dso_spatial_result , dso_spatial_parsermemo = :dso_spatial_parsermemo , dso_center_native = :dso_center_native , dso_geo_bb_native = :dso_geo_bb_native , dso_geo_native = :dso_geo_native , dso_center_wgs84 = {0} , dso_geo_bb_wgs84 = {1} , dso_geo_wgs84 = {2} , dso_prj_wkt = :dso_prj_wkt , dso_prj_proj4 = :dso_prj_proj4 , dso_prj_project = :dso_prj_project , dso_prj_coordinate = :dso_prj_coordinate , dso_prj_degree = :dso_prj_degree , dso_prj_zone = :dso_prj_zone , dso_prj_source = :dso_prj_source where dsoid = :dsoid '''.format(dso_center_wgs84, dso_geo_bb_wgs84, dso_geo_wgs84), params ) return CResult.merge_result(self.Success, '空间元数据处理完毕!')
def process_mission(self, dataset): storage_id = dataset.value_by_name(0, 'root_directory_id', '') storage_root_path = dataset.value_by_name(0, 'root_directory', '') CLogger().debug('storage_id: {0}'.format(storage_id)) ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str( dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times)) self.update_status(storage_id, process_result, self.ProcStatus_Error) return process_result sql_check_root_storage_dir_exist = ''' select dsdid from dm2_storage_directory where dsdid = :dsdid ''' sql_update_root_storage_dir = ''' update dm2_storage_directory set dsdParentID = '-1', dsdDirectory = '', dsdDirtype = {1} , dsdDirectoryName = '', dsdPath = '' , dsdDirCreateTime = :dsddircreatetime, dsdDirLastModifyTime = :dsddirlastmodifytime , dsdLastModifyTime = Now(), dsd_directory_valid = {0} where dsdid = :dsdid '''.format(self.File_Status_Unknown, self.Dir_Type_Root) sql_insert_root_storage_dir = ''' insert into dm2_storage_directory( dsdid, dsdparentid, dsdstorageid, dsddirectory, dsddirtype, dsdlastmodifytime , dsddirectoryname, dsd_directory_valid, dsdpath, dsddircreatetime, dsddirlastmodifytime) values(:dsdid, '-1', :dsdStorageID, '', {1}, Now() , '', {0}, '', :dsddircreatetime, :dsddirlastmodifytime ) '''.format(self.File_Status_Unknown, self.Dir_Type_Root) try: db = CFactory().give_me_db(self.get_mission_db_id()) params = dict() params['dsdid'] = storage_id params['dsdStorageID'] = storage_id if CFile.file_or_path_exist(storage_root_path): params['dsdDirCreateTime'] = CFile.file_modify_time( storage_root_path) params['dsddirlastmodifytime'] = CFile.file_modify_time( storage_root_path) if db.if_exists(sql_check_root_storage_dir_exist, params): db.execute(sql_update_root_storage_dir, params) else: db.execute(sql_insert_root_storage_dir, params) process_result = CResult.merge_result(CResult.Success, '存储扫描处理成功') self.update_status(storage_id, process_result) return process_result except DBException as err: process_result = CResult.merge_result( CResult.Exception, '存储扫描失败, 原因为{0}'.format(err.__str__)) self.update_status(storage_id, process_result) return process_result
def open_source(self) -> CDataSetSeqReader: option_source = CUtils.dict_value_by_name(self.__config, self.Name_Source, None) if option_source is None: raise Exception('数据源未设置, 请检查修正后重试! ') type_source = CUtils.dict_value_by_name(option_source, self.Name_Type, self.Name_Table) if CUtils.equal_ignore_case(type_source, self.Name_ShapeFile): source_data_option = CUtils.dict_value_by_name(option_source, type_source, None) source_data_filename = CUtils.dict_value_by_name(source_data_option, self.Name_Name, None) if CUtils.equal_ignore_case(source_data_filename, ''): raise Exception('[{0}]类型数据源未设置文件名, 请检查修正后重试! '.format(type_source)) if not CFile.file_or_path_exist(source_data_filename): raise Exception('[{0}]类型数据源设置的文件[{1}]不存在, 请检查修正后重试! '.format(type_source, source_data_filename)) cpg_file_name = CFile.change_file_ext(source_data_filename, 'cpg') encoding = CResource.Encoding_GBK if CFile.file_or_path_exist(cpg_file_name): encoding = CFile.file_2_str(cpg_file_name) self.__source_data = CVectorDataSets(source_data_filename) self.__source_data.open(encoding) return self.__source_data.layer_by_index(0) elif CUtils.equal_ignore_case(type_source, self.Name_Vector_DataSet): source_data_option = CUtils.dict_value_by_name(option_source, type_source, None) source_data_filename = CUtils.dict_value_by_name(source_data_option, self.Name_Name, None) source_data_layer = CUtils.dict_value_by_name(source_data_option, self.Name_Layer, None) if CUtils.equal_ignore_case(source_data_filename, ''): raise Exception('[{0}]类型数据源未设置文件名, 请检查修正后重试! '.format(type_source)) if CUtils.equal_ignore_case(source_data_layer, ''): raise Exception('[{0}]类型数据源未设置图层名, 请检查修正后重试! '.format(type_source)) if not CFile.file_or_path_exist(source_data_filename): raise Exception('[{0}]类型数据源设置的文件[{1}]不存在, 请检查修正后重试! '.format(type_source, source_data_filename)) self.__source_data = CVectorDataSets(source_data_filename) self.__source_data.open() layer_data = self.__source_data.layer_by_name(source_data_layer) if layer_data is None: raise Exception('[{0}]类型数据源设置的图层[{1}]不存在, 请检查修正后重试! '.format(type_source, source_data_layer)) return layer_data elif CUtils.equal_ignore_case(type_source, self.Name_Table): source_data_option = CUtils.dict_value_by_name(option_source, type_source, None) source_database = CUtils.dict_value_by_name(source_data_option, self.Name_DataBase, self.DB_Server_ID_Default) source_data_table_name = CUtils.dict_value_by_name(source_data_option, self.Name_Name, None) if CUtils.equal_ignore_case(source_data_table_name, ''): raise Exception('[{0}]类型数据源未设置数据表名, 请检查修正后重试! '.format(self.Name_Table)) data_set = CFactory().give_me_db(source_database).all_row( 'select * from {0}'.format(self.replace_placeholder(source_data_table_name)), self.__params ) self.__source_data = CDBDataSetSeqReader(data_set) return self.__source_data elif CUtils.equal_ignore_case(type_source, self.Name_Table): source_data_option = CUtils.dict_value_by_name(option_source, type_source, None) source_database = CUtils.dict_value_by_name(source_data_option, self.Name_DataBase, self.DB_Server_ID_Default) source_data_sql = CUtils.dict_value_by_name(source_data_option, self.Name_SQL, None) if CUtils.equal_ignore_case(source_data_sql, ''): raise Exception('[{0}]类型数据源未设置查询语句, 请检查修正后重试! '.format(type_source)) data_set = CFactory().give_me_db(source_database).all_row( self.replace_placeholder(source_data_sql), self.__params ) self.__source_data = CDBDataSetSeqReader(data_set) return self.__source_data else: raise Exception('系统不支持类型为[{0}]的数据源, 请检查修正后重试! '.format(type_source))
def process_mission(self, dataset) -> str: """ :param dataset: :return: """ ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '') ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_storage_title = dataset.value_by_name(0, 'query_storage_title', '') ds_storage_root_dir = dataset.value_by_name(0, 'query_rootpath', '') ds_ib_directory_name = dataset.value_by_name(0, 'query_ib_relation_dir', '') ds_ib_directory_id = dataset.value_by_name(0, 'query_ib_relation_dir_id', '') ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '') # 按需要再开启 # ds_ib_option = CUtils.any_2_str(dataset.value_by_name(0, 'query_ib_option', '')) if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format( ds_storage_title, CFile.join_file(ds_storage_root_dir, ds_ib_directory_name))) else: CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format( ds_storage_title, ds_storage_root_dir)) try: ds_ib_information_updated = False # 检查目录名格式并自动修正 if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): ds_ib_directory = CFile.unify( CFile.add_prefix(ds_ib_directory_name)) if not CUtils.equal_ignore_case(ds_ib_directory, ds_ib_directory_name): ds_ib_directory_name = ds_ib_directory ds_ib_information_updated = True if CUtils.equal_ignore_case(ds_ib_batch_no, ''): ds_ib_batch_no = CFactory().give_me_db( self.get_mission_db_id()).seq_next_value( self.Seq_Type_Date_AutoInc) ds_ib_information_updated = True if CUtils.equal_ignore_case(ds_ib_directory_id, ''): ds_ib_directory_id = CUtils.one_id() ds_ib_information_updated = True if ds_ib_information_updated: self.correct_ib_information(ds_ib_id, ds_ib_directory_name, ds_ib_batch_no, ds_ib_directory_id) if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): ib_full_directory = CFile.join_file(ds_storage_root_dir, ds_ib_directory_name) else: ib_full_directory = ds_storage_root_dir self.clear_anything_in_directory(ds_ib_id) metadata_rule_file_name = CFile.join_file( ib_full_directory, self.FileName_MetaData_Rule) metadata_rule_content = '' if CFile.file_or_path_exist(metadata_rule_file_name): try: metadata_rule_content = CXml.file_2_str( metadata_rule_file_name) CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format( ib_full_directory, metadata_rule_content)) except Exception as error: result = CResult.merge_result( self.Failure, '在目录[{0}]下发现元数据规则文件, 但它的格式不合法, 详细错误为: [{1}]'.format( ib_full_directory, error.__str__())) self.update_inbound_qi_result(ds_ib_id, result) return result path_obj = CDMPathInfo(self.FileType_Dir, ib_full_directory, ds_storage_id, ds_ib_directory_id, ds_storage_id, None, self.get_mission_db_id(), metadata_rule_content) if path_obj.white_black_valid(): path_obj.db_check_and_update(ds_ib_id) result = CResult.merge_result( self.Success, '目录[{0}]的入库质检任务创建成功, 系统正在质检, 请稍后...'.format( ib_full_directory)) else: result = CResult.merge_result( self.Failure, '目录[{0}]未通过黑白名单检验, 不允许入库! '.format(ib_full_directory)) self.update_inbound_qi_result(ds_ib_id, result) return result except Exception as error: result = CResult.merge_result( self.Failure, '目录[{0}]的入库质检任务创建过程出现错误, 详细错误为: [{1}]'.format( CFile.join_file(ds_storage_root_dir, ds_ib_directory_name), error.__str__())) self.update_inbound_qi_result(ds_ib_id, result) return result
def db_check_and_update_metadata_rule(self, metadata_rule_file_name): """ 检查并判断指定的元数据扫描规则文件是否与数据库中的记录相等 1. 如果和记录中的不同 删除当前目录下的所有子目录, 文件 和对象 更新记录中的规则 设置子目录扫描状态为正常 2. 如果和记录中的相同 返回 :param metadata_rule_file_name: :return: """ metadata_rule_content = '' if CFile.file_or_path_exist(metadata_rule_file_name): try: metadata_rule_content = CXml.file_2_str( metadata_rule_file_name) CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format( self.file_name_with_full_path, metadata_rule_content)) except: pass db_metadata_rule_content = self.ds_file_or_path.value_by_name( 0, 'dsdscanrule', '') CLogger().debug('目录[{0}]在库中登记的规则内容为[{1}]'.format( self.file_name_with_full_path, db_metadata_rule_content)) if CUtils.equal_ignore_case(metadata_rule_content, db_metadata_rule_content): CLogger().debug('目录[{0}]的规则内容与库中的相同'.format( self.file_name_with_full_path)) return else: CLogger().debug('目录[{0}]的规则内容与库中的不同, 将清理目录下的文件, 重新入库!'.format( self.file_name_with_full_path)) sql_update_path_scan_rule = ''' update dm2_storage_directory set dsd_directory_valid = -1, dsdscanrule = :dsdScanRule where dsdid = :dsdID ''' sql_clear_files_of_path = ''' delete from dm2_storage_file where dsfdirectoryid in ( select dsdid from dm2_storage_directory where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1 ) ''' sql_clear_subpath_of_path = ''' delete from dm2_storage_directory where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1 and dsdid <> :dsdID ''' engine = CFactory().give_me_db(self.db_server_id) session = engine.give_me_session() try: params = dict() params['dsdID'] = self.my_id if metadata_rule_content == '': params['dsdScanRule'] = None else: params['dsdScanRule'] = metadata_rule_content engine.session_execute(session, sql_update_path_scan_rule, params) params = dict() params['dsdID'] = self.my_id params['dsdStorageID'] = self.storage_id params['dsdSubDirectory'] = CFile.join_file( self.file_name_with_rel_path, '') engine.session_execute(session, sql_clear_files_of_path, params) engine.session_execute(session, sql_clear_subpath_of_path, params) engine.session_commit(session) except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__())) engine.session_rollback(session) finally: engine.session_close(session)
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 王学谦 :param parser: :return: """ file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name check_file_metadata_bus_exist = False ext = self.Transformer_XML metadata_name_with_path = CFile.join_file(file_path, '{0}M.xml'.format(file_main_name[:-1])) if CFile.file_or_path_exist(metadata_name_with_path): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = metadata_name_with_path if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' } ) else: parser.metadata.quality.append_total_quality( { self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format( CFile.file_name(self.metadata_bus_src_filename_with_path) ) } ) letter_location_FenFu = file_path.find('FenFu') file_path_sub = file_path[letter_location_FenFu + 6:] letter_location = file_path_sub.find(CFile.sep()) shp_path = CFile.join_file(file_path[:letter_location_FenFu + 5], file_path_sub[:letter_location]) shp_list = CFile.file_or_subpath_of_path(shp_path, '(?i).shp$', CFile.MatchType_Regex) if len(shp_list) == 0: parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'shp_file', self.Name_Title: '影像时相接边图', self.Name_Result: self.QA_Result_Warn, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少影像时相接边图' } ) else: parser.metadata.quality.append_total_quality( { self.Name_FileName: shp_list[0], self.Name_ID: 'shp_file', self.Name_Title: '影像时相接边图', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '影像时相接边图[{0}]存在'.format(shp_list[0]) } )
if layer_name.upper().endswith('_SHAPE_INDEX'): return False elif CUtils.equal_ignore_case(layer_name, 'Selections'): return False elif CUtils.equal_ignore_case(layer_name, 'SelectedObjects'): return False return True if __name__ == '__main__': file_name = '/Users/wangxiya/Documents/我的测试数据/31.混合存储/测试数据/通用数据/矢量数据集/生态治理和水土保持监测数据库_黑岱沟露天煤矿_10017699_2020d1_2020-01-01.mdb' # file_name = '/Users/wangxiya/Documents/我的测试数据/31.混合存储/测试数据/通用数据/矢量/矢量数据/正确数据/生态治理和水土保持监测数据库_利民煤矿_10017574_2020d1_2020-08-05/取排口.shp' cpg_file_name = CFile.change_file_ext(file_name, 'cpg') encoding = CResource.Encoding_GBK if CFile.file_or_path_exist(cpg_file_name): encoding = CFile.file_2_str(cpg_file_name) vector_datasets = CVectorDataSets(file_name) vector_datasets.open(encoding) for vector_layer_index in range(vector_datasets.size()): vector_dataset = vector_datasets.layer_by_index(vector_layer_index) print('*' * 30) print('{0}-{1}'.format(vector_dataset.dataset_name, vector_dataset.size())) print('*' * 10) for field_index in range(vector_dataset.field_count()): print('{0}.{1}'.format(vector_dataset.dataset_name, vector_dataset.field_name(field_index))) vector_dataset = vector_datasets.layer_by_index(0) print('*' * 30)
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 :param parser: :return: """ super().qa_file_custom(parser) file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name file_name_before_six = file_main_name[0:6] # 截取前六位行政区划代码 # 查询行政区划代码对应的名称并拼接 try: db = CFactory().give_me_db(self.file_info.db_server_id) file_name_before_six_name = db.one_row("select gdstitle from ro_global_dim_space " "where gdsid = '{0}'".format(file_name_before_six)) \ .value_by_name(0, 'gdstitle', None) except: file_name_before_six_name = '' file_metadata_name = '{0}{1}'.format(file_name_before_six, file_name_before_six_name) check_file_metadata_bus_exist = False ext = self.Transformer_DOM_MDB # 后缀名 metadata_main_name_with_path = CFile.join_file(file_path, file_metadata_name) # 拼接好元数据文件名称,并检查其是否存在 temp_metadata_bus_file = '{0}.{1}'.format(metadata_main_name_with_path, ext) if CFile.file_or_path_exist(temp_metadata_bus_file): check_file_metadata_bus_exist = True self.metadata_bus_transformer_type = ext self.metadata_bus_src_filename_with_path = temp_metadata_bus_file if not check_file_metadata_bus_exist: parser.metadata.quality.append_total_quality({ self.Name_FileName: '', self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本文件缺少业务元数据' }) else: parser.metadata.quality.append_total_quality({ self.Name_FileName: self.metadata_bus_src_filename_with_path, self.Name_ID: 'metadata_file', self.Name_Title: '元数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path) })
def create_virtual_content(self) -> bool: CFile.check_and_create_directory_itself(self.__work_root_dir__) return CFile.file_or_path_exist(self.__work_root_dir__)
def create_virtual_content(self) -> bool: return super().create_virtual_content() and CFile.file_or_path_exist( self.__virtual_content_root_dir__)
def __find_module_obj(self, obj_id): sql_query = ''' SELECT dm2_storage_file.dsfid as query_file_id, dm2_storage_file.dsfdirectoryid as query_directory_id, dm2_storage_directory.dsddirlastmodifytime as query_directory_lastmodifytime, dm2_storage_directory.dsdid as query_dataset_directory_id, dm2_storage_object.* FROM dm2_storage_object LEFT JOIN dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid LEFT JOIN dm2_storage_directory on dm2_storage_directory.dsd_object_id = dm2_storage_object.dsoid WHERE dm2_storage_object.dsoid = '{0}' '''.format(obj_id) db_id = self._db_id # 数据库连接标识(查询的dm2的数管库) dataset = CFactory().give_me_db(db_id).one_row(sql_query) object_id = dataset.value_by_name(0, 'dsoid', '') object_name = dataset.value_by_name(0, 'dsoobjectname', '') quality_info = dataset.value_by_name(0, 'dso_quality', '') quality_info_xml = CXml() quality_info_xml.load_xml(quality_info) # 加载查询出来的xml db_id_distribution = self.DB_Server_ID_Distribution # 同步处理的目标数据库标识id # 构建数据对象object对应的识别插件,获取get_information里面的Plugins_Info_Module_Distribute_Engine信息 class_classified_obj = CObject.get_plugins_instance_by_object_id( db_id, object_id) if class_classified_obj is not None: plugins_info = class_classified_obj.get_information() obj_type_code = CUtils.dict_value_by_name( plugins_info, class_classified_obj.Plugins_Info_Type_Code, '') distribution_file_main_name = CUtils.dict_value_by_name( plugins_info, class_classified_obj.Plugins_Info_Module_Distribute_Engine, '') # 对应同步文件的类名称 # 判断同步插件文件是否存在 access_modules_root_dir = CSys.get_metadata_data_access_modules_root_dir( ) distribution_root_dir = CFile.join_file(access_modules_root_dir, self.Name_Distribution) distribution_file = CFile.join_file( distribution_root_dir, '{0}.py'.format(distribution_file_main_name)) if CFile.file_or_path_exist(distribution_file): # 构建同步对象 distribution_obj = CObject.create_module_distribution_instance( '{0}.{1}'.format( CSys.get_metadata_data_access_modules_root_name(), self.Name_Distribution), distribution_file_main_name, db_id_distribution, object_id, object_name, obj_type_code, quality_info_xml, dataset) distribution_obj.set_class_plugins(class_classified_obj) result = CResult.merge_result(self.Success, '数据的信息提取完成') return distribution_obj, result else: result = CResult.merge_result(self.Failure, '同步失败,因为本类数据的同步模块未建立') return None, result else: result = CResult.merge_result(self.Failure, '同步失败,因为本类数据的识别模块缺失') return None, result
def virtual_content_valid(self) -> bool: return CFile.file_or_path_exist(self.__virtual_content_root_dir__)