def get_sync_dict_list(self, insert_or_updata) -> list: """ insert_or_updata 中 self.DB_True为insert,DB_False为updata 本方法的写法为强规则,调用add_value_to_sync_dict_list配置 第一个参数为list,第二个参数为字段名,第三个参数为字段值,第四个参数为特殊配置 """ object_name = self._obj_name dsometadataxml_bus = self._dataset.value_by_name( 0, 'dsometadataxml_bus', '') xml = CXml() xml.load_xml(dsometadataxml_bus) # 初始化xml dsometadataxml_bus_type = '{0}'.format( xml.xpath_one("/root/@type")) # 查询业务元数据类别 if object_name is not None: if dsometadataxml_bus_type is not None: if CUtils.equal_ignore_case(dsometadataxml_bus_type, 'mdb'): return self.get_sync_mdb_dict_list(insert_or_updata) elif CUtils.equal_ignore_case(dsometadataxml_bus_type, 'mat'): return self.get_sync_mat_dict_list(insert_or_updata) elif CUtils.equal_ignore_case(dsometadataxml_bus_type, 'xls') \ or CUtils.equal_ignore_case(dsometadataxml_bus_type, 'xlsx'): return self.get_sync_xls_dict_list(insert_or_updata) raise Exception("数据{0}业务元数据类型为'{1}'出现错误,没有可识别的类型".format( object_name, dsometadataxml_bus_type))
def _layer_is_dataset(self, layer_obj) -> bool: """ 判断一个层是否是合法的数据集 :param layer_obj: :return: """ driver = self._data_obj.GetDriver() if driver is None: return True layer_name = CUtils.any_2_str(layer_obj.GetName()) if CUtils.equal_ignore_case(driver.name, 'OpenFileGDB'): if layer_name.upper().startswith('T_1_'): return False elif CUtils.equal_ignore_case(driver.name, 'PGeo'): if layer_name.upper().endswith('_SHAPE_INDEX'): return False elif CUtils.equal_ignore_case(layer_name, 'Selections'): return False elif CUtils.equal_ignore_case(layer_name, 'SelectedObjects'): return False return True
def classified(self): """ 设计国土行业数据custom的验证规则(自定义影像) 完成 负责人 李宪 在这里检验custom的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext file_object_name = file_main_name file_name_with_full_path = self.file_info.file_name_with_full_path # 初始化需要的参数 if file_name_with_full_path.endswith('_21at.xml'): file_object_name = file_main_name[:-5] file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) check_file_main_name_exist_tif = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) check_file_main_name_exist_img = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Img)) if (not check_file_main_name_exist_tif) and ( not check_file_main_name_exist_img): return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \ or CUtils.equal_ignore_case(file_ext, self.Name_Img): self._object_confirm = self.Object_Confirm_IKnown self._object_name = self.file_info.file_main_name file_detail_xml = '{0}_21at.xml'.format( self.file_info.file_main_name_with_full_path) self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中 else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def metadata_bus_dict_process_custom(self, metadata_bus_dict): """ 对部分需要进行运算的数据进行处理 """ super().metadata_bus_dict_process_custom(metadata_bus_dict) resolution = CUtils.dict_value_by_name(metadata_bus_dict, 'resolution', None) if not CUtils.equal_ignore_case(resolution, ''): resolution_list = re.split(r'[/]|\s+', resolution.strip()) if len(resolution_list) > 0: temp_resolution_list = [] for temp_resolution in resolution_list: temp_resolution_list.append( CUtils.to_decimal(temp_resolution)) metadata_bus_dict['resolution'] = min(temp_resolution_list)
def classified(self): """ 设计国土行业数据的dom-10验证规则 完成 负责人 李宪 在这里检验dom-10的元数据文件格式时, 应该一个一个类型的对比, 找到文件时, 将该文件的格式和文件名存储到类的私有属性中, 以便在元数据处理时直接使用 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext check_file_main_name_length = len(file_main_name) == 10 if not check_file_main_name_length: return self.Object_Confirm_IUnKnown, self._object_name file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name) check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format( file_main_name_with_path, self.Name_Tif)) if not check_file_main_name_exist: return self.Object_Confirm_IUnKnown, self._object_name """ 下面判别第1位是字母 下面判别第4位是字母 下面判别第23位是数字 下面判别第567位是数字 下面判别第8910位是数字 """ char_1 = file_main_name[0:1] char_2_3 = file_main_name[1:3] char_4 = file_main_name[3:4] char_5_to_7 = file_main_name[4:7] char_8_to_10 = file_main_name[7:10] if CUtils.text_is_alpha(char_1) is False \ or CUtils.text_is_numeric(char_2_3) is False \ or CUtils.text_is_alpha(char_4) is False \ or CUtils.text_is_numeric(char_5_to_7) is False \ or CUtils.text_is_numeric(char_8_to_10) is False: return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(file_ext, self.Name_Tif): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def add_file_to_detail_list(self): """ 设定国土行业数据mosaic的附属文件的验证规则(镶嵌影像) 完成 负责人 李宪 在这里检验mosaic的附属文件的识别规则 :return: """ file_detail_xml = '{0}_21at.xml'.format( self.file_info.file_main_name_with_full_path) self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中 file_detail_xq = '{0}xq.*'.format(self.file_info.file_main_name) if not CUtils.equal_ignore_case(self.file_info.file_path, ''): list_file_fullname_xq = CFile.file_or_dir_fullname_of_path( self.file_info.file_path, False, file_detail_xq, CFile.MatchType_Common) # 模糊匹配文件列表 for list_file_fullname in list_file_fullname_xq: self.add_file_to_details(list_file_fullname) # 将文件加入到附属文件列表中
def db_access_check(self, access_wait_flag, access_forbid_flag, message): temporary_dict = dict() temporary_dict['dso_time'] = self._dataset.value_by_name( 0, 'dso_time', '') temporary_dict['dso_browser'] = self._dataset.value_by_name( 0, 'dso_browser', '') temporary_dict['dso_thumb'] = self._dataset.value_by_name( 0, 'dso_thumb', '') temporary_dict['dso_geo_wgs84'] = self._dataset.value_by_name( 0, 'dso_geo_wgs84', '') for key, value in temporary_dict.items(): if CUtils.equal_ignore_case(value, ''): message = message + '[数据{0}入库异常!请进行检查与修正!]'.format( key.replace('dso_', '')) access_forbid_flag = self.DB_True return access_wait_flag, access_forbid_flag, message
def __init__(self, file_type, file_name_with_full_path, root_path, rule_content): super().__init__(file_type, file_name_with_full_path) self.__root_path = root_path if CUtils.equal_ignore_case(root_path, file_name_with_full_path): self.__file_main_name_with_rel_path = '' self.__file_name_with_rel_path = '' self.__file_path_with_rel_path = '' else: self.__file_main_name_with_rel_path = CFile.file_relation_path( self.file_main_name_with_full_path, self.__root_path) self.__file_name_with_rel_path = CFile.file_relation_path( self.file_name_with_full_path, self.__root_path) self.__file_path_with_rel_path = CFile.file_relation_path( self.file_path, self.__root_path) self.__rule_content = rule_content
def give_me_db(self, db_id=None) -> CDataBase: if db_id is None: rt_db_id = self.DB_Server_ID_Default else: rt_db_id = CUtils.any_2_str(db_id) if CUtils.equal_ignore_case(rt_db_id, ''): rt_db_id = self.DB_Server_ID_Default databases = settings.application.xpath_one(self.Name_DataBases, None) if databases is None: raise Exception('系统未配置数据库定义, 请检查修正后重试!') for database in databases: if rt_db_id == CUtils.dict_value_by_name(database, self.Name_ID, self.DB_Server_ID_Default): return self.create_db(database) raise Exception('未找到标示为[{0}]]的数据库定义!'.format(rt_db_id))
def get_ib_schema(self, dataset_type, ib_option): """ 根据数据集类型, 获取系统设置中的最佳入库模式 :param ib_option: 其他入库的特殊要求 :param dataset_type: :return: """ ib_schema_list = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_Schema_Special, []) ib_schema_default = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_Schema_Default, None) for ib_schema in ib_schema_list: ib_schema_id = CUtils.dict_value_by_name(ib_schema, self.Name_ID, '') if CUtils.equal_ignore_case(ib_schema_id, dataset_type): return ib_schema else: return ib_schema_default
def seq_next_value(self, seq_type: int) -> str: if seq_type == self.Seq_Type_Date_AutoInc: sql_last_seq_date = ''' select coalesce(gcfgvalue, current_date::text) as last_date, current_date::text as curr_date from ro_global_config where gcfgcode = 'sys_seq_date_autoinc' ''' ds_last_seq_date = self.one_row(sql_last_seq_date) if ds_last_seq_date.is_empty(): self.execute(''' insert into ro_global_config(gcfgid, gcfgcode, gcfgtitle, gcfgvalue, gcfgmemo) values (10001, 'sys_seq_date_autoinc', '日期自增序列最后记录', current_date::text, null) ''') ds_last_seq_date = self.one_row(sql_last_seq_date) last_seq_date = ds_last_seq_date.value_by_name( 0, 'last_date', None) curr_seq_date = ds_last_seq_date.value_by_name( 0, 'curr_date', None) if last_seq_date is None: self.execute(''' update ro_global_config set gcfgvalue = current_date::text where gcfgcode = 'sys_seq_date_autoinc' ''') elif not CUtils.equal_ignore_case(last_seq_date, curr_seq_date): self.execute(''' update ro_global_config set gcfgvalue = current_date::text where gcfgcode = 'sys_seq_date_autoinc' ''') next_value = self.one_row( "select setval('sys_seq_date_autoinc', 1)").value_by_index( 0, 0, 1) return '{0}-{1}'.format(curr_seq_date, CUtils.int_2_format_str(next_value, 3)) next_value = self.one_row( "select nextval('sys_seq_date_autoinc')").value_by_index( 0, 0, 1) return '{0}-{1}'.format(curr_seq_date, CUtils.int_2_format_str(next_value, 3)) else: return CUtils.any_2_str( self.one_row( "select nextval('sys_seq_autoinc')").value_by_index( 0, 0, 1))
def create_job_instance(cls, package_directory, package_root_name, package_subdir, package_name, *args, **kwargs): package_root = os.path.join(package_directory, package_subdir) dir_list = os.listdir(package_root) for cur_file in dir_list: if not CUtils.equal_ignore_case(CFile.file_main_name(cur_file), package_name): continue path = os.path.join(package_root, cur_file) if os.path.isfile(path) and (not cur_file.startswith('__init__')): package_root_name = '{0}.{1}.{2}'.format(package_root_name, package_subdir, package_name) package_obj = importlib.import_module(package_root_name) class_meta = getattr(package_obj, package_name) class_meta_one = class_meta obj_id = args[0] obj_params = args[1] obj = class_meta_one(obj_id, obj_params) return obj else: raise BusinessNotExistException(package_subdir, package_name)
def db_check_and_update(self, ib_id): """ 检查并更新dm2_storage_file表中记录 :return: """ if not self.ds_file_or_path.is_empty(): # 如果记录已经存在 db_file_modify_time = self.ds_file_or_path.value_by_name(0, 'dsddirlastmodifytime', '') db_file_size = self.ds_file_or_path.value_by_name(0, 'dsffilesize', 0) if CUtils.equal_ignore_case(CUtils.any_2_str(db_file_modify_time), CUtils.any_2_str(self.file_modify_time)) and ( db_file_size == self.file_size): CLogger().info('文件[{0}]的大小和最后修改时间, 和库中登记的没有变化, 文件将被设置为忽略刷新! '.format(self.file_name_with_full_path)) CFactory().give_me_db(self.db_server_id).execute( ''' update dm2_storage_file set dsfScanStatus = 0, dsffilevalid = -1, dsf_ib_id = :ib_id, dsfscanmemo = :message where dsfid = :dsfid ''', { 'dsfid': self.my_id, 'ib_id': ib_id, 'message': '文件[{0}]的大小和最后修改时间, 和库中登记的没有变化, 文件将被设置为忽略刷新! '.format(self.file_name_with_full_path) } ) else: CLogger().info('文件[{0}]的大小和最后修改时间, 和库中登记的有变化, 文件将被设置为重新刷新! '.format(self.file_name_with_full_path)) CFactory().give_me_db(self.db_server_id).execute( ''' update dm2_storage_file set dsfScanStatus = 1, dsffilevalid = -1, dsf_ib_id = :ib_id, dsfscanmemo = :message where dsfid = :dsfid ''', { 'dsfid': self.my_id, 'ib_id': ib_id, 'message': '文件[{0}]的大小和最后修改时间, 和库中登记的有变化, 文件将被设置为重新刷新! '.format(self.file_name_with_full_path) } ) else: CLogger().info('文件[{0}]未在库中登记, 系统将登记该记录! '.format(self.file_name_with_full_path)) self.__db_insert(ib_id)
def get_mdb_connect(self, file_metadata_name_with_path): """ :return: """ try: # win下需要安装AccessDatabaseEngine_X64.exe驱动 # linux下需要安装mdbtools与unixODBC-2.3.9.tar与gdal,gdal拥有pgeo驱动 # 并将odbcinst.ini文件的设置一个连接名为Driver=Microsoft Access Driver (*.mdb)的项目 mdb = 'Driver=Microsoft Access Driver (*.mdb);' + 'DBQ={0}'.format( file_metadata_name_with_path ) # win驱动,安装AccessDatabaseEngine_X64.exe驱动 # get_os_name方法返回值, posix, nt, java对应linux / windows / java虚拟机 if CUtils.equal_ignore_case(CSys.get_os_name(), 'nt'): # win下返回nt conn = pypyodbc.win_connect_mdb( mdb) # 安装pypyodbc插件,本插件为python写的,可全平台 else: conn = pypyodbc.connect(mdb) except Exception as error: raise Exception('mdb解析驱动异常:' + error.__str__()) return conn
def classified(self): file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 初始化需要的参数 file_object_name = file_main_name[:] file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) if CUtils.equal_ignore_case(file_ext, self.Name_Shp): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name else: if CFile.file_or_path_exist('{0}.{1}'.format( file_main_name_with_path, self.Name_Shp)): self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None else: self._object_confirm = self.Object_Confirm_IUnKnown self._object_name = None return self._object_confirm, self._object_name
def process(self) -> str: """ :return: """ super().process() file_metadata_name_with_path = self.transformer_src_filename try: xml_obj = None if CUtils.equal_ignore_case(self.transformer_type, self.Transformer_DOM_MDB): xml_obj = self.mdb_to_xml(file_metadata_name_with_path) else: raise if xml_obj is not None: super().metadata.set_metadata_bus( self.Success, '元数据文件[{0}]成功加载! '.format(file_metadata_name_with_path), self.MetaDataFormat_XML, xml_obj.to_xml() ) return CResult.merge_result( self.Success, '元数据文件[{0}]成功加载! '.format(file_metadata_name_with_path) ) else: raise except Exception as error: super().metadata.set_metadata_bus( self.Exception, '元数据文件[{0}]格式不合法, 无法处理! 错误原因为{1}' .format(file_metadata_name_with_path, error.__str__()), self.MetaDataFormat_Text, '' ) return CResult.merge_result( self.Exception, '元数据文件[{0}]格式不合法, 无法处理! 错误原因为{1}' .format(file_metadata_name_with_path, error.__str__()) )
def sync(self, object_access, obj_id, obj_name, obj_type, quality) -> str: """ 处理数管中识别的对象, 与第三方模块的同步 . 如果第三方模块自行处理, 则无需继承本方法 . 如果第三方模块可以处理, 则在本模块中, 从数据库中提取对象的信息, 写入第三方模块的数据表中, 或者调用第三方模块接口 注意: 在本方法中, 不要用_quality_info属性, 因为外部调用方考虑的效率因素, 没有传入!!! :return: """ # 根据objecttype类型查找distribution文件夹下对应的类文件(识别通过objecttype找object_def表中的dsodtype字段与类对象中的info[self.Name_Type]值相同) if CUtils.equal_ignore_case(self.DataAccess_Pass, object_access): distribution_obj_real, result = self.__find_module_obj(obj_id) if not CResult.result_success(result): return result elif distribution_obj_real is None: message = '没有对应的算法, 直接通过!' result = CResult.merge_result(self.Success, message) return result result = distribution_obj_real.sync() return result else: # todo(赵宇飞) 这里如果权限是等待审批或禁用, 则从ap3系列表中删除记录 return CResult.merge_result(self.Success, '这里需要从ap3系列表中删除记录, 等待实现!')
def process_vector(self) -> str: try: # file_name_with_full_path = r'D:\test\vector_test\石嘴山市-3xq.json' # file_main_name = CFile.file_main_name(file_name_with_full_path) # file_path = CFile.file_path(file_name_with_full_path) xml_obj = self.metadata.metadata_xml() # <editor-fold desc="1.空间坐标信息"> wkt_info = 'POLYGON((${min_x} ${max_y},${max_x} ${max_y},' \ '${max_x} ${min_y},${min_x} ${min_y},${min_x} ${max_y}))' # 四至坐标 native_max_x = CUtils.to_decimal( xml_obj.get_element_text_by_xpath_one('/TileMetadata/MaxLon')) native_max_y = CUtils.to_decimal( xml_obj.get_element_text_by_xpath_one('/TileMetadata/MaxLat')) native_min_x = CUtils.to_decimal( xml_obj.get_element_text_by_xpath_one('/TileMetadata/MinLon')) native_min_y = CUtils.to_decimal( xml_obj.get_element_text_by_xpath_one('/TileMetadata/MinLat')) if (native_max_x is None) or (native_max_y is None) \ or (native_min_x is None) or (native_min_y is None): native_center_wkt = None native_bbox_wkt = None geom_native_wkt = None else: dict_native = { 'max_x': CUtils.any_2_str(native_max_x), 'max_y': CUtils.any_2_str(native_max_y), 'min_x': CUtils.any_2_str(native_min_x), 'min_y': CUtils.any_2_str(native_min_y) } # 中心坐标 center_x = (native_max_x - native_min_x) / 2 + native_min_x center_y = (native_max_y - native_min_y) / 2 + native_min_y native_center_wkt = 'POINT({0} {1})'.format(center_x, center_y) # 外边框、外包框 native_bbox_wkt = CUtils.replace_placeholder( wkt_info[:], dict_native) geom_native_wkt = native_bbox_wkt file_path = self.file_content.work_root_dir file_main_name = self.object_name native_center_filepath = CFile.join_file( file_path, file_main_name + '_native_center.wkt') CFile.str_2_file(native_center_wkt, native_center_filepath) native_bbox_filepath = CFile.join_file( file_path, file_main_name + '_native_bbox.wkt') CFile.str_2_file(native_bbox_wkt, native_bbox_filepath) native_geom_filepath = CFile.join_file( file_path, file_main_name + '_native_geom.wkt') CFile.str_2_file(geom_native_wkt, native_geom_filepath) projection = xml_obj.get_element_text_by_xpath_one( '/TileMetadata/SpatialReference/PRJ') if (projection is not None) and (not CUtils.equal_ignore_case( projection, '')): source_projection = osr.SpatialReference(wkt=projection) source = source_projection.GetAttrValue('GEOGCS', 0) # 坐标系名称 prosrs = osr.SpatialReference() if prosrs.ImportFromWkt(projection) == gdal.CE_None: proj_wkt = prosrs.ExportToWkt() native_wkt = proj_wkt else: native_wkt = projection native_proj4 = prosrs.ExportToProj4() spatial = None rb = (0, 0) lu = (0, 0) geosrs = prosrs.CloneGeogCS() ct = osr.CreateCoordinateTransformation(prosrs, geosrs) if ct is not None: rb = ct.TransformPoint(native_max_x, native_max_y) lu = ct.TransformPoint(native_min_x, native_min_y) wgs84_min_x = lu[0] wgs84_max_y = lu[1] wgs84_max_x = rb[0] wgs84_min_y = rb[1] dict_wgs84 = { 'max_x': CUtils.any_2_str(wgs84_max_x), 'max_y': CUtils.any_2_str(wgs84_max_y), 'min_x': CUtils.any_2_str(wgs84_min_x), 'min_y': CUtils.any_2_str(wgs84_min_y) } # 中心坐标 center_x = (wgs84_max_x - wgs84_min_x) / 2 + wgs84_min_x center_y = (wgs84_max_y - wgs84_min_y) / 2 + wgs84_min_y wgs84_center_wkt = 'POINT({0} {1})'.format( center_x, center_y) # 外边框、外包框 wgs84_bbox_wkt = CUtils.replace_placeholder( wkt_info[:], dict_wgs84) wgs84_geom_wkt = wgs84_bbox_wkt wgs84_center_filepath = CFile.join_file( file_path, file_main_name + '_wgs84_center.wkt') CFile.str_2_file(wgs84_center_wkt, wgs84_center_filepath) wgs84_bbox_filepath = CFile.join_file( file_path, file_main_name + '_wgs84_bbox.wkt') CFile.str_2_file(wgs84_bbox_wkt, wgs84_bbox_filepath) wgs84_geom_filepath = CFile.join_file( file_path, file_main_name + '_wgs84_geom.wkt') CFile.str_2_file(wgs84_geom_wkt, wgs84_geom_filepath) # </editor-fold> # <editor-fold desc="2.投影信息"> native_source = CResource.Prj_Source_Data # 坐标系 native_coordinate = None # 3度带/6度带 native_degree = None # 投影方式 native_project = None # 带号 native_zone = None # 创建SpatialReference对象,导入wkt信息 spatial_ref = osr.SpatialReference(wkt=native_wkt) if spatial_ref.IsProjected(): native_project = spatial_ref.GetAttrValue('PROJECTION') native_coordinate = spatial_ref.GetAttrValue('GEOGCS') # native_degree = spatial_ref.GetAttrValue('GEOGCS|UNIT', 1) native_degree, native_zone = self.get_prj_degree_zone( spatial_ref) elif spatial_ref.IsGeocentric(): native_project = None native_coordinate = spatial_ref.GetAttrValue('GEOGCS') native_degree = None native_zone = None # </editor-fold> result = CResult.merge_result(self.Success, '处理完毕!') result = CResult.merge_result_info(result, self.Name_Prj_Wkt, native_wkt) result = CResult.merge_result_info(result, self.Name_Prj_Proj4, native_proj4) result = CResult.merge_result_info(result, self.Name_Prj_Project, native_project) result = CResult.merge_result_info(result, self.Name_Prj_Coordinate, native_coordinate) result = CResult.merge_result_info(result, self.Name_Prj_Source, native_source) result = CResult.merge_result_info(result, self.Name_Prj_Zone, native_zone) result = CResult.merge_result_info(result, self.Name_Prj_Degree, native_degree) return result # return CResult.merge_result(self.Success, '处理完毕!') else: return CResult.merge_result(self.Success, '处理完毕!') except Exception as error: CLogger().warning('矢量数据的空间信息处理出现异常, 错误信息为: {0}'.format( error.__str__())) return CResult.merge_result( self.Failure, '矢量数据的空间信息处理出现异常,错误信息为:{0}!'.format(error.__str__()))
def process_mission(self, dataset) -> str: ds_subpath = dataset.value_by_name(0, 'query_subpath', '') ds_root_path = dataset.value_by_name(0, 'query_root_path', '') ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_id = dataset.value_by_name(0, 'query_dir_id', '') owner_obj_id = dataset.value_by_name(0, 'query_dir_parent_objid', '') parent_id = dataset.value_by_name(0, 'query_dir_parent_id', '') ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str(dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times ) ) self.update_dir_status(ds_id, process_result, self.ProcStatus_Error) return process_result if CUtils.equal_ignore_case(ds_subpath, ''): result = CResult.merge_result(CResult.Success, '根目录[{0}]不支持识别为对象, 当前流程被忽略!') self.update_dir_status(ds_id, result) return result ds_path_full_name = CFile.join_file(ds_root_path, ds_subpath) CLogger().debug('处理的子目录为: {0}'.format(ds_path_full_name)) try: sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_storage_id, 'dsdDirectory': ds_subpath } ) ds_rule_content = rule_ds.value_by_name(0, 'dsdScanRule', '') path_obj = CDMPathInfo(self.FileType_Dir, ds_path_full_name, ds_storage_id, ds_id, parent_id, owner_obj_id, self.get_mission_db_id(), ds_rule_content) if not path_obj.file_existed: path_obj.db_update_status_on_path_invalid() return CResult.merge_result( CResult.Success, '目录[{0}]不存在, 在设定状态后, 顺利结束!'.format(ds_path_full_name) ) else: path_obj.db_check_and_update_metadata_rule( CFile.join_file(ds_path_full_name, self.FileName_MetaData_Rule) ) path_obj.db_path2object() result = CResult.merge_result(CResult.Success, '目录[{0}]处理顺利完成!'.format(ds_path_full_name)) self.update_dir_status(ds_id, result) return result except Exception as error: result = CResult.merge_result( CResult.Failure, '目录[{0}]识别过程出现错误, 详细情况: {1}!'.format(ds_path_full_name, error.__str__()) ) self.update_dir_status(ds_id, result) return result
def notify_object(self, inbound_id: str, access: str, memo: str, obj_id, obj_name, obj_type, quality) -> str: """ 处理数管中识别的对象, 与第三方模块的通知 . 如果第三方模块自行处理, 则无需继承本方法 . 如果第三方模块可以处理, 则在本模块中, 从数据库中提取对象的信息, 写入第三方模块的数据表中, 或者调用第三方模块接口 注意: 在本方法中, 不要用_quality_info属性, 因为外部调用方考虑的效率因素, 没有传入!!! @:param access 当前模块对当前对象的权限 :return: """ ds_na = CFactory().give_me_db(self._db_id).one_row( ''' select dsonid, dson_notify_status from dm2_storage_obj_na where dson_app_id = :app_id and dson_object_id = :object_id ''', { 'app_id': CUtils.dict_value_by_name(self.information(), self.Name_ID, ''), 'object_id': obj_id }) target_notify_status = self.ProcStatus_InQueue if not CUtils.equal_ignore_case(access, self.DataAccess_Pass): target_notify_status = self.ProcStatus_Finished if not ds_na.is_empty(): na_id = CUtils.any_2_str(ds_na.value_by_name(0, 'dsonid', 0)) if (ds_na.value_by_name(0, 'dson_notify_status', self.ProcStatus_Finished) == self.ProcStatus_Finished) or \ (ds_na.value_by_name(0, 'dson_notify_status', self.ProcStatus_Finished) == self.ProcStatus_InQueue): CFactory().give_me_db(self._db_id).execute( ''' update dm2_storage_obj_na set dson_notify_status = :status , dson_object_access = :object_access , dson_access_memo = :object_access_memo , dson_inbound_id = :inbound_id where dsonid = :id ''', { 'id': na_id, 'status': target_notify_status, 'object_access': access, 'inbound_id': inbound_id, 'object_access_memo': memo }) else: CFactory().give_me_db(self._db_id).execute( ''' insert into dm2_storage_obj_na(dson_app_id, dson_object_access, dson_object_id, dson_inbound_id, dson_access_memo) values(:app_id, :object_access, :object_id, :inbound_id, :object_access_memo) ''', { 'app_id': CUtils.dict_value_by_name(self.information(), self.Name_ID, ''), 'object_id': obj_id, 'object_access': access, 'inbound_id': inbound_id, 'object_access_memo': memo }) return CResult.merge_result( self.Success, '对象[{0}]已经推送给模块[{1}]队列! '.format( obj_name, CUtils.dict_value_by_name(self.information(), self.Name_Title, '')))
def plugins_classified(cls, file_info: CDMFilePathInfoEx) -> CPlugins: """ 插件识别 1. 首先检查file_info的__rule_content__中, 有无优先识别插件列表, 有则使用该列表按顺序进行识别 1. 其次, 检查file_info的__rule_content__是否为空, 如果不为空, 则获取其类型, 如果类型不为空, 则按类型, 匹配系统配置 1. 其次, 检查应用系统配置中, 有无对目录识别插件的特殊设置, 有则按设置中的列表进行识别 1. 最后按系统插件目录下的顺序, 对数据进行识别 :param file_info: :return: """ class_classified_obj = None # 根据rule_content, 获取入库规则文件的特定插件列表 if file_info.file_type == cls.FileType_Dir: plugin_node_list = CXml.xml_xpath(file_info.rule_content, cls.Path_MD_Rule_Plugins_Dir) else: plugin_node_list = CXml.xml_xpath(file_info.rule_content, cls.Path_MD_Rule_Plugins_File) if len(plugin_node_list) > 0: # 根据指定的插件列表, 尝试进行识别 class_classified_obj = cls.__plugins_classified_by_plugin_node_list__( file_info, plugin_node_list) if class_classified_obj is not None: return class_classified_obj # 如果入库规则文件中没有指定插件列表, 则读取数据入库规则类型, 它==数据集类型 rule_type = CXml.get_element_text( CXml.xml_xpath_one(file_info.rule_content, cls.Path_MD_Rule_Type)) if not CUtils.equal_ignore_case(rule_type, ''): # 如果有数据入库规则文件, 则获取其规则类型, 匹配setting中的类型 plugins_json_array = settings.application.xpath_one( cls.Path_Setting_MetaData_Plugins_Dir, None) if plugins_json_array is not None: for plugins_define in plugins_json_array: key_word = CUtils.any_2_str( CUtils.dict_value_by_name(plugins_define, cls.Name_Keyword, None)) plugin_list = CUtils.dict_value_by_name( plugins_define, cls.Name_Plugin, None) if plugin_list is None: continue if CUtils.equal_ignore_case(key_word, rule_type): class_classified_obj = cls.__plugins_classified_by_plugin_list__( file_info, plugin_list) else: # 如果没有数据入库规则文件, 则通过目录, 逐一匹配setting中的类型 file_path = file_info.file_path_with_rel_path plugins_json_array = settings.application.xpath_one( cls.Path_Setting_MetaData_Plugins_Dir, None) if plugins_json_array is not None: for plugins_define in plugins_json_array: key_word = CUtils.any_2_str( CUtils.dict_value_by_name(plugins_define, cls.Name_Keyword, None)) plugin_list = CUtils.dict_value_by_name( plugins_define, cls.Name_Plugin, None) if plugin_list is None: continue # todo(注意) 如果关键字为空, 则表明所有子目录都优先使用设置的插件列表进行识别!!! if CUtils.equal_ignore_case(key_word, ''): class_classified_obj = cls.__plugins_classified_by_plugin_list__( file_info, plugin_list) else: if CFile.subpath_in_path(CUtils.any_2_str(key_word), file_path): class_classified_obj = cls.__plugins_classified_by_plugin_list__( file_info, plugin_list) if class_classified_obj is not None: return class_classified_obj else: continue if class_classified_obj is not None: return class_classified_obj else: return cls.__plugins_classified_of_directory__(file_info)
def image2view(self, target_ds, view_path: str) -> str: """ 影像文件转jpg或png 对波段数据类型为Byte(8位无符号整型)的影像,可采用gdal.Translate()方法直接进行文件格式转换。 对波段数据类型为UInt16(16位无符号整型)、Int16(16位有符号整型)的影像: 由于JPEG不支持16位影像的转换,且PNG转换效果非常不理想,图像轮廓模糊。 因此对16位影像采用百分比截断的方法压缩至0~255的范围,改变了像素深度,降低到8位。 该方法生成的快视图的转换效果与gdal.Translate()方法生成的相比,图像轮廓清晰,可辨识度高。 注:gdal == 3.1.3 :param target_ds: 影像临时文件的数据集 :param view_path: 快视图或拇指图文件地址 :return: """ cols = target_ds.RasterXSize rows = target_ds.RasterYSize band_count = target_ds.RasterCount band = target_ds.GetRasterBand(1) data_type_name = gdal.GetDataTypeName(band.DataType) # 检查影像位深度(波段数据类型) if CUtils.equal_ignore_case(data_type_name, 'Byte'): # 对波段数据类型为Byte(8位无符号整型)的影像进行转换 if CFile.check_and_create_directory(view_path): if CFile.file_ext(view_path).lower() == 'jpg': out = gdal.Translate(view_path, target_ds, format='JPEG') elif CFile.file_ext(view_path).lower() == 'png': out = gdal.Translate(view_path, target_ds, format='PNG') else: # 对波段数据类型为UInt16(16位无符号整型)、Int16(16位有符号整型)的影像进行转换 # 检查影像波段数并读取 if band_count >= 3: bandsOrder = [3, 2, 1] data = np.empty([rows, cols, 3], dtype=float) for i in range(3): band = target_ds.GetRasterBand(bandsOrder[i]) data1 = band.ReadAsArray(0, 0, cols, rows) data[:, :, i] = data1 if CFile.check_and_create_directory(view_path): # 百分比截断压缩影像,将像元取值限定在0~255 lower_percent = 0.6 higher_percent = 99.4 n = data.shape[2] out = np.zeros_like(data, dtype=np.uint8) for i in range(n): a = 0 b = 255 c = np.percentile(data[:, :, i], lower_percent) d = np.percentile(data[:, :, i], higher_percent) t = a + (data[:, :, i] - c) * (b - a) / (d - c) t[t < a] = a t[t > b] = b out[:, :, i] = t outImg = Image.fromarray(np.uint8(out)) outImg.save(view_path) else: data = np.empty([rows, cols], dtype=float) band = target_ds.GetRasterBand(1) data1 = band.ReadAsArray(0, 0, cols, rows) data[:, :] = data1 if CFile.check_and_create_directory(view_path): # 百分比截断压缩影像,将像元取值限定在0~255 lower_percent = 0.6 higher_percent = 99.4 out = np.zeros_like(data, dtype=np.uint8) a = 0 b = 255 c = np.percentile(data[:, :], lower_percent) d = np.percentile(data[:, :], higher_percent) t = a + (data[:, :] - c) * (b - a) / (d - c) t[t < a] = a t[t > b] = b out[:, :] = t outImg = Image.fromarray(np.uint8(out)) outImg.save(view_path) del target_ds del out return view_path
def init_qa_metadata_bus_xml_list(self, parser: CMetaDataParser) -> list: """ 初始化默认的, 业务元数据xml文件的检验列表 完成 负责人 王学谦 :param parser: :return: """ qa_metadata_bus_xml_list = [{ self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//MetaDataFileName", self.Name_ID: 'MetaDataFileName', self.Name_Title: '带扩展名元数据文件名', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_NotNull: True, self.Name_DataType: self.value_type_string, self.Name_Width: 60 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/ProductName", self.Name_ID: 'ProductName', self.Name_Title: '对象名称', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_NotNull: True, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/Owner", self.Name_ID: 'Owner', self.Name_Title: '所有者', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/Producer", self.Name_ID: 'Producer', self.Name_Title: '生产商', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/Publisher", self.Name_ID: 'Publisher', self.Name_Title: '出版商', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/ProduceDate", self.Name_ID: 'ProduceDate', self.Name_Title: '生产日期', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_date, self.Name_NotNull: True }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/ConfidentialLevel", self.Name_ID: 'ConfidentialLevel', self.Name_Title: '密级', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/GroundResolution", self.Name_ID: 'GroundResolution', self.Name_Title: '地面分辨率', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_NotNull: True, self.Name_DataType: self.value_type_string, self.Name_Width: 10 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/ImgColorModel", self.Name_ID: 'ImgColorModel', self.Name_Title: '影像类型', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/PixelBits", self.Name_ID: 'PixelBits', self.Name_Title: '位深', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 50 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/DataFormat", self.Name_ID: 'DataFormat', self.Name_Title: '格式', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_NotNull: True, self.Name_DataType: self.value_type_string, self.Name_Width: 25 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "/Metadatafile/BasicDataContent/Mathfoundation/MapProjection", self.Name_ID: 'MapProjection', self.Name_Title: '地图投影', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 50 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//SateName", self.Name_ID: 'SateName', self.Name_Title: '星源', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 38 }] try: imgtype = parser.metadata.metadata_bus_xml( ).get_element_text_by_xpath_one('//ImgSource') except Exception: imgtype = None if CUtils.equal_ignore_case(imgtype, '0'): qa_metadata_bus_xml_list.extend([{ self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//CameraType", self.Name_ID: 'CameraType', self.Name_Title: '航摄仪型号', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//DigitalPhotoResolution", self.Name_ID: 'DigitalPhotoResolution', self.Name_Title: '航片分辨率', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_decimal_or_integer, self.Name_Width: 8 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//PhotoDate", self.Name_ID: 'PhotoDate', self.Name_Title: '航片日期', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_date }]) elif CUtils.equal_ignore_case(imgtype, '1'): qa_metadata_bus_xml_list.extend([{ self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//PBandSensorType", self.Name_ID: 'PBandSensorType', self.Name_Title: '全色传感器', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//SateResolution", self.Name_ID: 'SateResolution', self.Name_Title: '全色分辨率', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_decimal_or_integer, self.Name_Width: 8 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//PbandOrbitCode", self.Name_ID: 'PBandOribitCode', self.Name_Title: '全色轨道号', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 38 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//PbandDate", self.Name_ID: 'PbandDate', self.Name_Title: '全色拍摄日期', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_date }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//MultiBandSensorType", self.Name_ID: 'MultiBandSensorType', self.Name_Title: '多光谱传感器或航摄仪型号', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 100 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//MultiBandResolution", self.Name_ID: 'MultiBandResolution', self.Name_Title: '多光谱分辨率', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_decimal_or_integer, self.Name_Width: 8 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//MultiBandOrbitCode", self.Name_ID: 'MultiBandOrbitCode', self.Name_Title: '多光谱轨道号', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 100 }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//MultiBandDate", self.Name_ID: 'MultiBandDate', self.Name_Title: '多光谱拍摄日期', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_decimal_or_integer }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//MultiBandNum", self.Name_ID: 'MultiBandNum', self.Name_Title: '多光谱波段数量', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_decimal_or_integer }, { self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//MultiBandName", self.Name_ID: 'MultiBandName', self.Name_Title: '多光谱波段名称', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_DataType: self.value_type_string, self.Name_Width: 20 }]) else: qa_metadata_bus_xml_list.extend([{ self.Name_Type: self.QA_Type_XML_Node_Exist, self.Name_XPath: "//ImgSource", self.Name_ID: 'ImgSource', self.Name_Title: '影像数据源类型', self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Result: self.QA_Result_Error, self.Name_NotNull: True, self.Name_List: ['0', '1'] }]) return qa_metadata_bus_xml_list
def _do_access(self) -> str: try: quality_info_xml = self._quality_info # 获取质检xml quality_summary = self._dataset.value_by_name(0, 'dso_quality_summary', '') quality_summary_json = CJson() quality_summary_json.load_obj(quality_summary) access_wait_flag = self.DB_False # 定义等待标志,为True则存在检查项目为等待 access_forbid_flag = self.DB_False # 定义禁止标志,为True则存在检查项目为禁止 message = '' # 文件与影像质检部分 file_qa = quality_summary_json.xpath_one('total', '') image_qa = quality_summary_json.xpath_one('metadata.data', '') if CUtils.equal_ignore_case(file_qa, self.QA_Result_Error) \ or CUtils.equal_ignore_case(image_qa, self.QA_Result_Error): message = message + '[数据与其相关文件的质检存在error!请进行修正!]' access_forbid_flag = self.DB_True elif CUtils.equal_ignore_case(file_qa, self.QA_Result_Warn) \ or CUtils.equal_ignore_case(image_qa, self.QA_Result_Warn): message = message + '[数据与其相关文件的质检存在warn!请进行检查!]' access_wait_flag = self.DB_True elif CUtils.equal_ignore_case(quality_summary, ''): message = message + '[数据质检未进行,可能数据存在问题!请进行检查!]' access_forbid_flag = self.DB_True else: pass for qa_name, qa_id in self.access_check_dict().items(): # 循环写好的检查列表 # qa_id = CUtils.dict_value_by_name(access_check_dict, 'qa_id', '') # 获取id qa_node = quality_info_xml.xpath_one("//item[@id='{0}']".format(qa_id)) # 查询xml中的节点 if qa_node is not None: node_result = CXml.get_attr(qa_node, self.Name_Result, '', False) # 获取质检结果 if CUtils.equal_ignore_case(node_result, self.QA_Result_Pass): pass elif CUtils.equal_ignore_case(node_result, self.QA_Result_Warn): # 警告则等待 message = message + '[业务元数据的质检中,项目{0}不符合要求,建议修正!]'.format(qa_name) access_wait_flag = self.DB_True else: # 错误以及其他情况,比如'',或者为其他字段 message = message + '[业务元数据的质检中,项目{0}不符合要求,必须修改后方可入库!]'.format(qa_name) access_forbid_flag = self.DB_True else: message = message + '[业务元数据的质检中,没有项目{0},请进行修正!]'.format(qa_name) access_forbid_flag = self.DB_True # 数据库部分 access_wait_flag, access_forbid_flag, message = \ self.db_access_check(access_wait_flag, access_forbid_flag, message) # 开始进行检查的结果判断 access_flag = self.DataAccess_Pass if access_forbid_flag: access_flag = self.DataAccess_Forbid elif access_wait_flag: access_flag = self.DataAccess_Wait if CUtils.equal_ignore_case(message, ''): message = '模块可以进行访问!' result = CResult.merge_result( self.Success, '模块[{0}.{1}]对对象[{2}]的访问能力已经分析完毕!分析结果为:{3}'.format( CUtils.dict_value_by_name(self.information(), self.Name_ID, ''), CUtils.dict_value_by_name(self.information(), self.Name_Title, ''), self._obj_name, message ) ) return CResult.merge_result_info(result, self.Name_Access, access_flag) except Exception as error: result = CResult.merge_result( self.Failure, '模块[{0}.{1}]对对象[{2}]的访问能力的分析存在异常!详细情况: {3}!'.format( CUtils.dict_value_by_name(self.information(), self.Name_ID, ''), CUtils.dict_value_by_name(self.information(), self.Name_Title, ''), self._obj_name, error.__str__() ) ) return CResult.merge_result_info(result, self.Name_Access, self.DataAccess_Forbid)
def classified(self): """ 关键字识别 """ super().classified() # 预获取需要的参数 file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 预定义逻辑参数 数据文件匹配 object_file_name_flag = False object_file_path_flag = False object_file_ext_flag = False object_affiliated_file_main_flag = False object_file_affiliated_flag = False object_keyword_list = self.get_classified_character_of_object_keyword() if len(object_keyword_list) > 0: for keyword_info in object_keyword_list: keyword_id = CUtils.dict_value_by_name(keyword_info, self.Name_ID, None) regex_match = CUtils.dict_value_by_name( keyword_info, self.Name_RegularExpression, '.*') if regex_match is None: regex_match = '.*' if CUtils.equal_ignore_case(keyword_id, self.Name_FileName): if CUtils.text_match_re(file_main_name, regex_match): object_file_name_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath): if CUtils.text_match_re(file_path, regex_match): object_file_path_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt): if CUtils.text_match_re(file_ext, regex_match): object_file_ext_flag = True else: same_name_file_list = CFile.file_or_dir_fullname_of_path( file_path, False, '(?i)^' + file_main_name + '[.].*$', CFile.MatchType_Regex) if len(same_name_file_list) > 0: for same_name_file in same_name_file_list: same_name_file_ext = CFile.file_ext( same_name_file) if CUtils.text_match_re( same_name_file_ext, regex_match): object_affiliated_file_main_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileAffiliated): affiliated_file_path = CUtils.dict_value_by_name( keyword_info, self.Name_FilePath, None) if affiliated_file_path is not None: if CFile.find_file_or_subpath_of_path( affiliated_file_path, regex_match, CFile.MatchType_Regex): object_file_affiliated_flag = True else: object_file_affiliated_flag = True # 预定义逻辑参数 附属文件匹配 affiliated_file_name_flag = False affiliated_file_path_flag = False affiliated_file_ext_flag = False affiliated_file_main_flag = False affiliated_keyword_list = self.get_classified_character_of_affiliated_keyword( ) if len(affiliated_keyword_list) > 0: for keyword_info in affiliated_keyword_list: keyword_id = CUtils.dict_value_by_name(keyword_info, self.Name_ID, None) regex_match = CUtils.dict_value_by_name( keyword_info, self.Name_RegularExpression, '.*') if regex_match is None: regex_match = '.*' if CUtils.equal_ignore_case(keyword_id, self.Name_FileName): if CUtils.text_match_re(file_main_name, regex_match): affiliated_file_name_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath): if CUtils.text_match_re(file_path, regex_match): affiliated_file_path_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt): if CUtils.text_match_re(file_ext, regex_match): affiliated_file_ext_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileMain): affiliated_file_path = CUtils.dict_value_by_name( keyword_info, self.Name_FilePath, None) if affiliated_file_path is not None: if CFile.find_file_or_subpath_of_path( affiliated_file_path, regex_match, CFile.MatchType_Regex): affiliated_file_main_flag = True if object_file_name_flag and object_file_path_flag and \ object_file_ext_flag and object_file_affiliated_flag: self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name self.set_custom_affiliated_file() elif affiliated_file_name_flag and affiliated_file_path_flag and \ affiliated_file_ext_flag and affiliated_file_main_flag: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None elif object_file_name_flag and object_file_path_flag and object_affiliated_file_main_flag: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None else: self._object_confirm = self.Object_Confirm_IUnKnown self._object_name = None return self._object_confirm, self._object_name
def search(self, module_name: str, search_json_obj: CJson, other_option: dict = None) -> CDataSet: """ 根据搜索条件json, 检索符合要求的对象, 并以数据集的方式返回如下字段: 1. object_id 1. object_name 1. object_type 1. object_data_type 1. object_parent_id 1. object_size 1. object_lastmodifytime :param module_name: 模块名称 :param search_json_obj: :param other_option: :return: """ if search_json_obj is None: return CDataSet() params_search = dict() sql_from = '' sql_where = '' if (not CUtils.equal_ignore_case(module_name, self.ModuleName_MetaData)) and \ (not CUtils.equal_ignore_case(module_name, '')): # sql_where = "dm2_storage_object.dso_da_result#>>'{{{0},result}}'='pass'".format(module_name) sql_from = ', dm2_storage_obj_na ' sql_where = " dm2_storage_obj_na.dson_app_id = 'module_name' " condition_obj_access = search_json_obj.xpath_one(self.Name_Access, self.DataAccess_Pass) if not CUtils.equal_ignore_case(condition_obj_access, ''): condition = "dm2_storage_obj_na.dson_object_access = '{0}'".format(CUtils.any_2_str(condition_obj_access)) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_inbound_id = search_json_obj.xpath_one(self.Name_InBound, None) if not CUtils.equal_ignore_case(condition_inbound_id, ''): condition = "dm2_storage_obj.dso_ib_id = '{0}'".format(CUtils.any_2_str(condition_inbound_id)) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_tag = search_json_obj.xpath_one(self.Name_Tag, None) if condition_tag is not None: if isinstance(condition_tag, list): condition = CUtils.list_2_str(condition_tag, "'", ", ", "'", True) else: condition = CUtils.list_2_str([condition_tag], "'", ", ", "'", True) if not CUtils.equal_ignore_case(condition, ''): condition = 'dm2_storage_object.dsotags @ > array[{0}]:: CHARACTER VARYING[]'.format(condition) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_id = search_json_obj.xpath_one(self.Name_ID, None) if condition_id is not None: if isinstance(condition_id, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodid', condition_id, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodid', condition_id, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_name = search_json_obj.xpath_one(self.Name_Name, None) if condition_name is not None: if isinstance(condition_name, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodname', condition_name, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodname', condition_name, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_type = search_json_obj.xpath_one(self.Name_Type, None) if condition_type is not None: if isinstance(condition_type, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodtype', condition_type, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodtype', condition_type, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_group = search_json_obj.xpath_one(self.Name_Group, None) if condition_group is not None: if isinstance(condition_group, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') if not CUtils.equal_ignore_case(sql_where, ''): sql_where = ' and {0}'.format(sql_where) sql_search = ''' select dm2_storage_object.dsoid as object_id , dm2_storage_object.dsoobjectname as object_name , dm2_storage_object.dsoobjecttype as object_type , dm2_storage_object.dsodatatype as object_data_type , dm2_storage_object.dsoparentobjid as object_parent_id , dm2_storage_object.dso_volumn_now as object_size , dm2_storage_object.dso_obj_lastmodifytime as object_lastmodifytime from dm2_storage_object, dm2_storage_object_def {0} where dm2_storage_object.dsoobjecttype = dm2_storage_object_def.dsodid and dm2_storage_object.dsoid = dm2_storage_obj_na.dson_object_id {1} '''.format(sql_from, sql_where) return CFactory().give_me_db(self.db_server_id).all_row(sql_search)
def __condition_value_like_2_sql(self, field, value): if CUtils.equal_ignore_case(value, ''): return "" return "{0} like '{0}'".format(field, value)
def parser_metadata_custom(self, parser: CMetaDataParser) -> str: """ 自定义的元数据解析, 在所有质检和其他处理之后触发 :param parser: :return: """ meta_data_json = parser.metadata.metadata_json() if meta_data_json is None: return CResult.merge_result( self.Success, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 无法获取JSON格式的元数据! '.format( self.file_info.file_name_with_full_path, ) ) json_data_source = meta_data_json.xpath_one('datasource', None) layer_list = meta_data_json.xpath_one(self.Name_Layers, None) if layer_list is None: return CResult.merge_result( self.Success, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 元数据中无法找到layers节点! '.format( self.file_info.file_name_with_full_path, ) ) mdb_ib_id = CFactory().give_me_db(self.file_info.db_server_id).one_value( ''' select dso_ib_id from dm2_storage_object where dsoid = :object_id ''', { 'object_id': parser.object_id } ) error_message_list = [] table = CTable() table.load_info(self.file_info.db_server_id, self.TableName_DM_Object) for layer in layer_list: layer_name = CUtils.dict_value_by_name(layer, self.Name_Name, '') if CUtils.equal_ignore_case(layer_name, ''): continue layer_alias_name = CUtils.dict_value_by_name(layer, self.Name_Description, layer_name) layer_metadata_json = CJson() layer_metadata_json.set_value_of_name('datasource', json_data_source) layer_metadata_json.set_value_of_name('layer_count', 1) layer_metadata_json.set_value_of_name('layers', [layer]) layer_metadata_text = layer_metadata_json.to_json() try: sql_find_layer_existed = ''' select dsoid as layer_id_existed from dm2_storage_object where upper(dsoobjectname) = upper(:layer_name) and dsoparentobjid = :object_id ''' layer_id_existed = CFactory().give_me_db(self.file_info.db_server_id).one_value( sql_find_layer_existed, { 'layer_name': layer_name, 'object_id': parser.object_id } ) if layer_id_existed is None: layer_id_existed = CUtils.one_id() table.column_list.reset() table.column_list.column_by_name('dsoid').set_value(layer_id_existed) table.column_list.column_by_name('dsoobjectname').set_value(layer_name) table.column_list.column_by_name('dsoobjecttype').set_value( CUtils.dict_value_by_name( self.get_information(), self.Plugins_Info_Child_Layer_Plugins_Name, '' ) ) table.column_list.column_by_name('dsodatatype').set_value( CUtils.dict_value_by_name( self.get_information(), self.Plugins_Info_Child_Layer_Data_Type, '' ) ) table.column_list.column_by_name('dsoalphacode').set_value(CUtils.alpha_text(layer_name)) table.column_list.column_by_name('dsoaliasname').set_value(layer_alias_name) table.column_list.column_by_name('dsoparentobjid').set_value(parser.object_id) table.column_list.column_by_name('dso_ib_id').set_value(mdb_ib_id) table.column_list.column_by_name('dsometadatatext').set_value(layer_metadata_text) table.column_list.column_by_name('dsometadatajson').set_value(layer_metadata_text) table.column_list.column_by_name('dsometadataparsestatus').set_value(self.ProcStatus_InQueue) table.column_list.column_by_name('dsotagsparsestatus').set_value(self.ProcStatus_InQueue) table.column_list.column_by_name('dsodetailparsestatus').set_value(self.ProcStatus_InQueue) result = table.save_data() if not CResult.result_success(result): error_message_list.append( '图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format( layer_name, CResult.result_message(result) ) ) except Exception as error: error_message_list.append('图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(layer_name, error.__str__())) if len(error_message_list) > 0: return CResult.merge_result( self.Failure, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 详细情况如下: \n{1}'.format( self.file_info.file_name_with_full_path, CUtils.list_2_str(error_message_list, '', '\n', '', True) ) ) else: return CResult.merge_result( self.Success, '数据[{0}]的自定义元数据解析完毕! '.format( self.file_info.file_name_with_full_path, ) )