def add_file_to_detail_list(self, match_name): """ 设定国土行业数据国情的附属文件的验证规则(镶嵌影像) 完成 负责人 王学谦 在这里检验国情的附属文件 :return: """ file_main_name = self._object_name file_path = self.file_info.file_path # 正则匹配附属文件 if not CUtils.equal_ignore_case(file_path, ''): match_str = '{0}*.*'.format(match_name) match_file_list = CFile.file_or_dir_fullname_of_path( file_path, False, match_str, CFile.MatchType_Common) match_str_main_name = r'(?i)^{0}[FMP]$'.format(match_name) # 主附属 ext_list = [ 'rar', 'zip', 'doc', 'docx', 'xls', 'xlsx', 'txt', 'xml' ] for file_with_path in match_file_list: if CUtils.equal_ignore_case( CFile.file_main_name(file_with_path), file_main_name): # 去除自身与同名文件 pass elif CUtils.text_match_re(CFile.file_main_name(file_with_path), match_str_main_name): self.add_file_to_details(file_with_path) # 将文件加入到附属文件列表中 elif CFile.file_ext(file_with_path).lower() in ext_list: self.add_file_to_details(file_with_path) else: pass
def get_vector_layercount_and_layers(self, datasource) -> (int, []): """ 获取矢量数据的图层个数和名称集合 @param datasource: @return: """ layer_count_real = 0 layer_list = [] driver = datasource.GetDriver() if driver is None: return layer_count_real, layer_list # gdb数据层里有一些是内置的拓扑检查的图层, 不要列入这部分为好 shp_ds = datasource iLayerCount = shp_ds.GetLayerCount() # print("iLayerCount:" + str(iLayerCount)) for i in range(iLayerCount): layer = shp_ds.GetLayer(i) layer_name = layer.GetName() # print('第{0}个图层:{1}'.format(i, layer_name)) if driver.name == 'OpenFileGDB': if layer_name.startswith('T_1_'): continue elif CUtils.equal_ignore_case(driver.name, 'PGeo'): if layer_name.upper().endswith('_SHAPE_INDEX'): continue elif CUtils.equal_ignore_case(layer_name, 'Selections'): continue elif CUtils.equal_ignore_case(layer_name, 'SelectedObjects'): continue layer_count_real = layer_count_real + 1 layer_list.append(layer) return layer_count_real, layer_list
def set_value_from_file(self, file_name: str, file_format: str, file_encoding: str): """ 设置value为文件内容 . 根据文件格式, 可以加载文件内容 . 如果文件格式为二进制, 则value存储文件名 :param file_name: :param file_format: :param file_encoding: :return: """ if CUtils.equal_ignore_case(file_format, CResource.FileFormat_TXT): self._value = { CResource.Name_Text: CFile.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_XML): self._value = { CResource.Name_Text: CXml.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_Json): self._value = { CResource.Name_Text: CJson.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } else: self._value = { CResource.Name_Text: file_name, CResource.Name_Type: CResource.DataValueType_File }
def check_src_ib_files_not_locked(self, root_path, parent_path): """ 检测指定目录下的文件是否没有被锁定 1. 只有所有文件都没有被锁定, 则返回True 1. 如果有任何一个文件被锁定, 则返回False, 而且把文件信息写入message中返回 todo(注意) 这里检查所有文件是否被锁定, 在处理切片数据时, 效率会极慢!!! :param root_path: 根目录 :param parent_path: 父目录, 在加入提示信息中时, 需要将父目录加入到反馈信息中 :return: 1. 目录下是否全部文件都没有锁定, 都可以入库 1. 被锁定文件的名称列表 """ parent_path = CFile.join_file(root_path, parent_path) locked_file_list = CFile.find_locked_file_in_path(parent_path) more_locked_file = False max_locked_file_count = len(locked_file_list) if max_locked_file_count > 3: max_locked_file_count = 3 more_locked_file = True message = '' for locked_file in locked_file_list: message = CUtils.str_append( message, CFile.join_file(parent_path, locked_file)) if more_locked_file: message = CUtils.str_append(message, '...') if max_locked_file_count > 0: message = CUtils.str_append(message, '被其他应用占用了, 无法入库, 请检查解除锁定后重试入库! ') return max_locked_file_count == 0, message
def classified(self): """ 设计国土行业数据ortho的验证规则(单景正射) 完成 负责人 王学谦 在这里检验ortho的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 初始化需要的参数 file_object_name = file_main_name[:] file_name_with_full_path = self.file_info.file_name_with_full_path if file_name_with_full_path.endswith('_21at.xml'): file_object_name = file_main_name[:-5] file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_object_name) check_file_main_name_exist = \ CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) or \ CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Img)) if not check_file_main_name_exist: # 检查主文件存在性 return self.Object_Confirm_IUnKnown, self._object_name # 检查后缀名 if CUtils.equal_ignore_case(file_ext, self.Name_Tif) or CUtils.equal_ignore_case(file_ext, self.Name_Img): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name file_detail_xml = '{0}_21at.xml'.format(self.file_info.file_main_name_with_full_path) self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中 else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def metadata_bus_dict_process_custom(self, metadata_bus_dict): """ 对部分需要进行运算的数据进行处理 """ super().metadata_bus_dict_process_custom(metadata_bus_dict) centerlatitude = CUtils.dict_value_by_name(metadata_bus_dict, 'centerlatitude', None) centerlongitude = CUtils.dict_value_by_name(metadata_bus_dict, 'centerlongitude', None) if (not CUtils.equal_ignore_case(centerlatitude, '')) and (not CUtils.equal_ignore_case(centerlongitude, '')): try: try: db_id = self.file_info.db_server_id except: db_id = self.DB_Server_ID_Distribution if CUtils.equal_ignore_case(db_id, ''): db_id = self.DB_Server_ID_Distribution db = CFactory().give_me_db(db_id) wkt = db.one_row( ''' select st_astext(st_envelope(st_geomfromewkt(st_astext(st_buffer(st_geographyfromtext( 'POINT({0} {1})'), 5000))))) as wkt '''.format(centerlatitude, centerlongitude)).value_by_name(0, 'wkt', None) wkt = wkt.replace('POLYGON((', '').replace('))', '').strip() coordinates_list = re.split(r'[,]|\s+', wkt) metadata_bus_dict['bottomleftlatitude'] = coordinates_list[0] metadata_bus_dict['bottomleftlongitude'] = coordinates_list[1] metadata_bus_dict['topleftlatitude'] = coordinates_list[2] metadata_bus_dict['topleftlongitude'] = coordinates_list[3] metadata_bus_dict['toprightlatitude'] = coordinates_list[4] metadata_bus_dict['toprightlongitude'] = coordinates_list[5] metadata_bus_dict['bottomrightlatitude'] = coordinates_list[6] metadata_bus_dict['bottomrightlongitude'] = coordinates_list[7] except Exception as error: raise Exception('通过中心点坐标计算四至坐标时出现异常,详细原因为:{0}'.format(error.__str__()))
def txt_to_xml(self, file_metadata_name_with_path: str): """ 完成 王学谦 txt文件转xml,在函数外提前定义xml对象并获取父节点传入,函数会将通过父节点构造xml对象 by王学谦 :param file_metadata_name_with_path:查询的mat文件全名,带路径 :return xml_obj:将文件内容存储好的项目对象 """ text_list = CFile.file_2_list( file_metadata_name_with_path) # 获取mat文件作为列表 if (text_list is None) or len(text_list) == 0: raise Exception('元数据文件无法读取,请检查') # 如果获取的文件内容为空,则抛出异常 xml_obj = CXml() # 建立xml对象 node_root = xml_obj.new_xml('root') xml_obj.set_attr(node_root, self.Name_Type, self.Transformer_TXT) # 设置root节点与属性 for row_text in text_list: if not CUtils.equal_ignore_case(row_text, ''): row_list = re.split(r'=', row_text.strip()) # 利用正则表达式,根据一个或多个tab剪切字符 # item节点 node_item = xml_obj.create_element(node_root, 'item') xml_obj.set_attr(node_item, self.Name_Name, CUtils.any_2_str(row_list[0]).strip()) if len(row_list) > 1: item_value = CUtils.any_2_str(row_list[1].strip()) if item_value.startswith('"') and item_value.endswith('"'): item_value = item_value[1:-1] xml_obj.set_element_text(node_item, item_value) # 设置item节点与属性与内容 return xml_obj
def classified(self): self._object_confirm = self.Object_Confirm_IUnKnown self._object_name = None current_path = self.file_info.file_name_with_full_path metadata_file_name = CFile.join_file(current_path, self.FileName_MetaData_Bus_21AT) if CFile.file_or_path_exist(metadata_file_name): self.__bus_metadata_xml_file_name__ = metadata_file_name self.__metadata_xml_obj__ = CXml() try: self.__metadata_xml_obj__.load_file(metadata_file_name) self.__classified_object_type = CXml.get_element_text( self.__metadata_xml_obj__.xpath_one( self.Path_21AT_MD_Content_ProductType)) if CUtils.equal_ignore_case( self.__classified_object_type, CUtils.dict_value_by_name(self.get_information(), self.Plugins_Info_Type, None)): self._object_confirm = self.Object_Confirm_IKnown self._object_name = CXml.get_element_text( self.__metadata_xml_obj__.xpath_one( self.Path_21AT_MD_Content_ProductName)) except: self.__metadata_xml_obj__ = None CLogger().warning('发现文件{0}符合二十一世纪业务数据集标准, 但该文件格式有误, 无法打开! ') return self._object_confirm, self._object_name
def get_sync_dict_list(self, insert_or_updata) -> list: """ insert_or_updata 指明配置的是更新还是插入,-1时为插入,0为更新 本方法的写法为强规则,调用add_value_to_sync_dict_list配置 第一个参数为list,第二个参数为字段名,第三个参数为字段值,第四个参数为特殊配置 """ sync_dict_list = self.get_sync_predefined_dict_list(insert_or_updata) object_table_data = self._dataset # 时间信息 dso_time = object_table_data.value_by_name(0, 'dso_time', '') dso_time_json = CJson() dso_time_json.load_obj(dso_time) self.add_value_to_sync_dict_list( sync_dict_list, 'begdate', dso_time_json.xpath_one('start_time', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'enddate', dso_time_json.xpath_one('end_time', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'imagedate', CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', ''))) self.add_value_to_sync_dict_list( sync_dict_list, 'producetime', CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', ''))) self.add_value_to_sync_dict_list( sync_dict_list, 'imagedatetag', self.transform_time_to_imagedatetag( CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', '')))) if insert_or_updata: self.add_value_to_sync_dict_list(sync_dict_list, 'isdel', '1') return sync_dict_list
def __load_table_info(self): """ 在这里加载数据表的信息, 并初始化__column_list对象 :return: """ dict_table_info = self.__database.table_info(self.__table_name) table_column_list = CUtils.dict_value_by_name(dict_table_info, self.Name_Columns, None) self.__column_list.clear() if table_column_list is None: return for table_column in table_column_list: column_name = CUtils.dict_value_by_name(table_column, self.Name_Name, '') if CUtils.equal_ignore_case(column_name, ''): continue column_data_type = CUtils.dict_value_by_name( table_column, self.Name_DataType, '') column_is_primary_key = CUtils.dict_value_by_name( table_column, self.Name_PrimaryKey, 0) self.__column_list.add( CColumn(column_name, column_data_type, column_is_primary_key == self.DB_True))
def object_uni_full_name_by_id(self, object_id: str): result = CFactory().give_me_db(self.db_server_id).one_value( ''' select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name from dm2_storage_object, dm2_storage_file, dm2_storage where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id and dm2_storage_file.dsfstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{0}' and dm2_storage_object.dsoid = :object_id union select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name from dm2_storage_object, dm2_storage_directory, dm2_storage where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id and dm2_storage_directory.dsdstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{1}' and dm2_storage_object.dsoid = :object_id '''.format(self.FileType_File, self.FileType_Dir) , {'object_id': object_id} ) if result is not None: return result ds_object_info = CFactory().give_me_db(self.db_server_id).one_row( ''' select dsoobjectname, dsoparentobjid from dm2_storage_object where dsoid = :object_id and dsodatatype not in ('file', 'dir') ''' ) if ds_object_info.is_empty(): return None object_name = ds_object_info.value_by_index(0, 0, '') object_parent_id = ds_object_info.value_by_index(0, 1, '') if CUtils.equal_ignore_case(object_parent_id, ''): return None result = CFactory().give_me_db(self.db_server_id).one_value( ''' select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name from dm2_storage_object, dm2_storage_file, dm2_storage where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id and dm2_storage_file.dsfstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{0}' and dm2_storage_object.dsoid = :object_id union select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name from dm2_storage_object, dm2_storage_directory, dm2_storage where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id and dm2_storage_directory.dsdstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{1}' and dm2_storage_object.dsoid = :object_id '''.format(self.FileType_File, self.FileType_Dir) , {'object_id': object_parent_id} ) if result is not None: return CFile.join_file(CUtils.any_2_str(result), object_name) else: return None
def parser_metadata_spatial_after_qa(self, parser: CMetaDataParser): """ 在这里直接指定坐标系 """ result = super().parser_metadata_spatial_after_qa(parser) try: Prj_Project = CUtils.dict_value_by_name(self.get_information(), self.Plugins_Info_Coordinate_System, '') if not CUtils.equal_ignore_case(Prj_Project, ''): parser.metadata.set_metadata_spatial( self.DB_True, '元数据文件[{0}]成功加载! '.format(self.file_info.file_name_with_full_path), self.Spatial_MetaData_Type_Prj_Project, Prj_Project ) parser.metadata.set_metadata_spatial( self.DB_True, '元数据文件[{0}]成功加载! '.format(self.file_info.file_name_with_full_path), self.Spatial_MetaData_Type_Prj_Source, self.Prj_Source_Custom ) except Exception as error: parser.metadata.set_metadata_spatial( self.DB_False, '元数据文件[{0}]格式不合法, 无法处理! 详细错误为: {1}'.format(self.file_info.file_name_with_full_path, error.__str__()), self.MetaDataFormat_Text, '') return CResult.merge_result(self.Exception, '元数据文件[{0}]格式不合法, 无法处理! '.format( self.file_info.file_name_with_full_path)) return result
def set_custom_affiliated_file(self): custom_affiliated_file_list = self.get_custom_affiliated_file_character( ) if len(custom_affiliated_file_list) > 0: for affiliated_file_info in custom_affiliated_file_list: affiliated_file_path = CUtils.dict_value_by_name( affiliated_file_info, self.Name_FilePath, None) regex_match = CUtils.dict_value_by_name( affiliated_file_info, self.Name_RegularExpression, None) no_match = CUtils.dict_value_by_name( affiliated_file_info, self.Name_No_Match_RegularExpression, None) if (affiliated_file_path is not None) and (regex_match is not None): affiliated_file_name_list = CFile.file_or_dir_fullname_of_path( affiliated_file_path, False, regex_match, CFile.MatchType_Regex) # 模糊匹配文件列表 if len(affiliated_file_name_list) > 0: for affiliated_file_name in affiliated_file_name_list: if no_match is None: self._object_detail_file_full_name_list.append( affiliated_file_name) else: if not CUtils.text_match_re( CFile.file_name(affiliated_file_name), no_match): self._object_detail_file_full_name_list.append( affiliated_file_name)
def mat_to_xml(self, file_metadata_name_with_path: str): """ 完成 王学谦 mat文件转xml,在函数外提前定义xml对象并获取父节点传入,函数会将通过父节点构造xml对象 by王学谦 :param file_metadata_name_with_path:查询的mat文件全名,带路径 :return xml_obj:将文件内容存储好的项目对象 """ text_list = CFile.file_2_list( file_metadata_name_with_path) # 获取mat文件作为列表 if (text_list is None) or len(text_list) == 0 or CUtils.equal_ignore_case( CUtils.any_2_str(text_list), ''): raise # 如果获取的文件内容为空,则抛出异常 flag = False # 设置标志 xml_obj = CXml() # 建立xml对象 node_root = xml_obj.new_xml('root') xml_obj.set_attr(node_root, self.Name_Type, self.transformer_type) # 设置root节点与属性 for index, row_text in enumerate(text_list): if row_text.startswith('1\t'): # 从开头为1+tab键的行开始录入 flag = True row_list = re.split(r'\s+', row_text) # 利用正则表达式,根据一个或多个tab剪切字符 if flag: node_item = xml_obj.create_element(node_root, 'item') xml_obj.set_attr(node_item, self.Name_Name, CUtils.any_2_str(row_list[1]).lower()) xml_obj.set_element_text( node_item, CUtils.any_2_str(row_list[2].strip())) # 设置item节点与属性与内容 if not flag: raise Exception('文件内容异常,无法正常识别文件开头') # 如果未找到1+tab键开头,则抛出异常 return xml_obj
def classified(self): """ 设计国土行业数据的dem_part_2验证规则 完成 负责人 李宪 在这里检验dem_part_2的识别规则 :return: """ super().classified() file_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name) check_file_main_name_exist_tif = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) check_file_main_name_exist_bil = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Bil)) if (not check_file_main_name_exist_tif) and (not check_file_main_name_exist_bil): return self.Object_Confirm_IUnKnown, self._object_name # 判断是否有‘-’,并且为一个 if not self.file_info.file_main_name.count('-') == 1: return self.Object_Confirm_IUnKnown, self._object_name char_1 = self.file_info.file_main_name.split('-')[0] char_2 = self.file_info.file_main_name.split('-')[1] # char_1,char_2是否小数 if CUtils.text_is_decimal(char_1) is False \ or CUtils.text_is_decimal(char_2) is False: return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Tif) \ or CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Bil): self._object_confirm = self.Object_Confirm_IKnown self._object_name = self.file_info.file_main_name else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def classified(self): """ 设计国土行业数据的dem_noframe非分幅数据的验证规则 完成 负责人 李宪 在这里检验dem_noframe的识别规则 :return: """ super().classified() file_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name) check_file_main_name_exist_tif = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Tif)) check_file_main_name_exist_img = CFile.file_or_path_exist( '{0}.{1}'.format(file_main_name_with_path, self.Name_Img)) if (not check_file_main_name_exist_tif) and (not check_file_main_name_exist_img): return self.Object_Confirm_IUnKnown, self._object_name if CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Tif) \ or CUtils.equal_ignore_case(self.file_info.file_ext, self.Name_Img): self._object_confirm = self.Object_Confirm_IKnown self._object_name = self.file_info.file_main_name file_detail_xml = '{0}_21at.xml'.format(self.file_info.file_main_name_with_full_path) self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中 else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def add_file_to_detail_list(self, match_name): """ 设定国土行业数据三调的附属文件的验证规则(镶嵌影像) 完成 负责人 王学谦 在这里检验三调的附属文件 :return: """ file_main_name = self._object_name file_path = self.file_info.file_path # 模糊匹配附属文件 if not CUtils.equal_ignore_case(file_path, ''): match_str = '{0}*xq.*'.format(match_name) match_file_list = CFile.file_or_dir_fullname_of_path( file_path, False, match_str, CFile.MatchType_Common) for file_with_path in match_file_list: if not CUtils.equal_ignore_case( CFile.file_main_name(file_with_path), file_main_name): # 去除自身与同名文件 self.add_file_to_details(file_with_path) # 将文件加入到附属文件列表中 try: db = CFactory().give_me_db(self.file_info.db_server_id) metadata_name_before_six_name = db.one_row("select gdstitle from ro_global_dim_space " "where gdsid = '{0}'".format(match_name)) \ .value_by_name(0, 'gdstitle', None) metadata_file_name = '{0}{1}.mdb'.format( match_name, metadata_name_before_six_name) metadata_file_name_with_path = CFile.join_file( file_path, metadata_file_name) if CFile.file_or_path_exist(metadata_file_name_with_path): self.add_file_to_details(metadata_file_name_with_path) except: pass
def search_type(self): """ 检索插件类型 :return: """ listplugin = [] plugins_root_dir = CSys.get_plugins_root_dir() plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir) for plugins_type in plugins_type_list: if CFile.is_dir(CFile.join_file( plugins_root_dir, plugins_type)) and ( not (str(plugins_type)).startswith('_')): plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), plugins_type) path = CFile.join_file(CSys.get_plugins_root_dir(), plugins_type) plugins_file_list = CFile.file_or_subpath_of_path( path, '{0}_*.{1}'.format(self.Name_Plugins, self.FileExt_Py)) for file_name_without_path in plugins_file_list: file_main_name = CFile.file_main_name( file_name_without_path) class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, file_main_name, None) plugins_info = class_classified_obj.get_information() # 获取插件的类型和名字 plugins_info["dsodid"] = '{0}'.format( plugins_type) + CFile.unify_seperator + '{0}'.format( file_main_name) listplugin.append(plugins_info) plugin_path = [] # 遍历listplugin for i in listplugin: file_dict = {} # 获取当前文件工作目录 work_path = CFile.file_abs_path('.') # 拼接通用路径 main_path = work_path + "/imetadata/business/metadata/inbound/plugins/" # 分割插件类型和名字 list = CUtils.dict_value_by_name(i, "dsodid", '').split(CFile.unify_seperator) # 拼接插件所在路径 file_path = main_path + CUtils.dict_value_by_name( i, "dsodid", '') + "." + self.FileExt_Py # 格式化文件路径 sorted_file_path = CFile.unify(file_path) """ type: dir/file/layer source: 待压缩的文件路径 target: 压缩后路径和名字(根据用户输入的压缩地址,然后拼接出完整的压缩文件) """ file_dict[CResource.Name_Type] = list[0] file_dict.setdefault(CResource.Name_Source, []).append(sorted_file_path) file_dict[CResource.Name_Target] = str( CUtils.dict_value_by_name(i, "dsodtype", '')) + ".zip" print(file_dict) plugin_path.append(file_dict) return plugin_path
def parser_file_or_subpath_of_path(self, dataset, ds_id, ds_path, ds_rule_content, inbound_id): """ 处理目录(完整路径)下的子目录和文件 :param inbound_id: 入库标识 :param ds_rule_content: :param dataset: 数据集 :param ds_id: 路径标识 :param ds_path: 路径全名 :return: """ ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ignore_file_array = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_ignore_file, None) ignore_dir_array = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_ignore_dir, None) file_list = CFile.file_or_subpath_of_path(ds_path) for file_name in file_list: file_name_with_full_path = CFile.join_file(ds_path, file_name) if CFile.is_dir(file_name_with_full_path): CLogger().debug('在目录{0}下发现子目录: {1}'.format(ds_path, file_name)) if CUtils.list_count(ignore_dir_array, file_name) > 0: CLogger().debug( '子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name)) continue path_obj = CDMPathInfo( self.FileType_Dir, file_name_with_full_path, dataset.value_by_name(0, 'query_storage_id', ''), None, ds_id, dataset.value_by_name(0, 'query_dir_parent_objid', None), self.get_mission_db_id(), ds_rule_content) if path_obj.white_black_valid(): path_obj.db_check_and_update(inbound_id) else: CLogger().info('目录[{0}]未通过黑白名单检验, 不允许入库! '.format( file_name_with_full_path)) elif CFile.is_file(file_name_with_full_path): if CUtils.list_count(ignore_file_array, file_name) > 0: CLogger().debug( '子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name)) continue CLogger().debug('在目录{0}下发现文件: {1}'.format(ds_path, file_name)) file_obj = CDMFileInfo( self.FileType_File, file_name_with_full_path, dataset.value_by_name(0, 'query_storage_id', ''), None, ds_id, dataset.value_by_name(0, 'query_dir_parent_objid', None), self.get_mission_db_id(), ds_rule_content) if file_obj.white_black_valid(): file_obj.db_check_and_update(inbound_id) else: CLogger().info('文件[{0}]未通过黑白名单检验, 不允许入库! '.format( file_name_with_full_path))
def record_as_dict(self) -> dict: data_record = self._data.record(self.__record_index) result = dict() key_list = data_record.keys() for key in key_list: result[CUtils.any_2_str(key).lower()] = CUtils.dict_value_by_name( data_record, key, None, False) return result
def json_set_attr(cls, json_text, attr_name: str, attr_value) -> str: new_result = CJson() rt_json_text = CUtils.any_2_str(json_text) if not CUtils.equal_ignore_case(rt_json_text, ''): new_result.load_json_text(rt_json_text) new_result.set_value_of_name(attr_name, attr_value) return new_result.to_json()
def join_file(cls, path: str, *paths: AnyStr) -> str: result = CUtils.any_2_str(path) for each_path in paths: real_file_name = CUtils.any_2_str(each_path) if real_file_name.startswith(r'/') or real_file_name.startswith('\\'): real_file_name = real_file_name[1:len(real_file_name)] result = '{0}{1}{2}'.format(result, cls.sep(), real_file_name) return result
def __inbound_object_detail_of_schema(self, list_file_fullname): sql_detail_insert = ''' INSERT INTO dm2_storage_obj_detail( dodid, dodobjectid, dodfilename, dodfileext, dodfilesize, dodfilecreatetime, dodfilemodifytime, dodlastmodifytime, dodfiletype) VALUES ( :dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize, :dodfilecreatetime, :dodfilemodifytime, now(), :dodfiletype) ''' sql_detail_insert_params_list = [] # query_storage_id = self.file_info.storage_id query_file_relation_name = self.file_info.file_name_with_rel_path for item_file_name_with_path in list_file_fullname: CLogger().debug(item_file_name_with_path) if not CFile.file_or_path_exist(item_file_name_with_path): continue params = dict() file_relation_name = CFile.file_relation_path( item_file_name_with_path, self.file_info.root_path) if CUtils.equal_ignore_case(query_file_relation_name, file_relation_name): params['dodid'] = self.object_id else: params['dodid'] = CUtils.one_id() # 文件类型 params['dodfiletype'] = self.FileType_File if CFile.is_dir(item_file_name_with_path): params['dodfiletype'] = self.FileType_Dir params['dodobjectid'] = self.object_id params['dodfilename'] = CFile.unify(file_relation_name) params['dodfileext'] = CFile.file_ext(item_file_name_with_path) params['dodfilesize'] = CFile.file_size(item_file_name_with_path) params['dodfilecreatetime'] = CFile.file_create_time( item_file_name_with_path) params['dodfilemodifytime'] = CFile.file_modify_time( item_file_name_with_path) # params['dodstorageid'] = query_storage_id # params['dodfilerelationname'] = CFile.file_relation_path( # item_file_name_with_path, # self.file_info.root_path) sql_params_tuple = (sql_detail_insert, params) sql_detail_insert_params_list.append(sql_params_tuple) if len(sql_detail_insert_params_list) > 0: try: CFactory().give_me_db( self.file_info.db_server_id).execute_batch( sql_detail_insert_params_list) except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format( error.__str__())) return CResult.merge_result(self.Failure, '处理失败!') return CResult.merge_result(self.Success, '处理完毕!')
def check_all_ib_file_or_path_existed(self, ib_id): """ 判断待入库数据的元数据, 与实体数据是否相符 . 返回CResult . 如果全部相符, 则返回True . 如果有任何一个不符, 则返回False, 且把不符的文件名通过信息返回 :param ib_id: :return: """ invalid_file_list = [] more_failure_file = False sql_all_ib_file = ''' select coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) || dm2_storage_file.dsffilerelationname as file_name , dm2_storage_file.dsffilesize as file_size , dm2_storage_file.dsffilemodifytime as file_modify_time from dm2_storage_file left join dm2_storage on dm2_storage.dstid = dm2_storage_file.dsfstorageid where dsf_ib_id = :ib_id ''' params_all_ib_file = {'ib_id': ib_id} ds_ib_file = CFactory().give_me_db(self.get_mission_db_id()).all_row( sql_all_ib_file, params_all_ib_file) for ds_ib_file_index in range(ds_ib_file.size()): file_valid = True file_name = ds_ib_file.value_by_name(ds_ib_file_index, 'file_name', '') if not CUtils.equal_ignore_case(file_name, ''): if not CFile.file_or_path_exist(file_name): file_valid = False elif not CUtils.equal_ignore_case( CFile.file_modify_time(file_name), ds_ib_file.value_by_name(ds_ib_file_index, 'file_modify_time', '')): file_valid = False elif CFile.file_size(file_name) != ds_ib_file.value_by_name( ds_ib_file_index, 'file_size', 0): file_valid = False if not file_valid: if len(invalid_file_list) <= 3: invalid_file_list.append(file_name) else: more_failure_file = True break if len(invalid_file_list) > 0: message = '' for invalid_file in invalid_file_list: message = CUtils.str_append(message, invalid_file) if more_failure_file: message = CUtils.str_append(message, '...') message = CUtils.str_append(message, '上述数据与库中记录不统一, 请重新扫描入库! ') return CResult.merge_result(self.Failure, message) else: return CResult.merge_result(self.Success, '所有文件均存在, 且与库中记录统一! ')
def register_dm_metadata_plugins(self): sql_register_dm_metadata_plugins_clear = ''' truncate table dm2_storage_object_def cascade ''' sql_unregister_dm_metadata_plugins = ''' delete from dm2_storage_object_def where dsodid = :dsodid ''' sql_register_dm_metadata_plugins = ''' insert into dm2_storage_object_def( dsodid, dsodtitle, dsodtype, dsodtypetitle, dsodtypecode, dsodgroup, dsodgrouptitle, dsodcatalog, dsodcatalogtitle, dsod_otheroption) values (:dsodid, :dsodtitle, :dsodtype, :dsodtypetitle, :dsodtypecode, :dsodgroup, :dsodgrouptitle, :dsodcatalog, :dsodcatalogtitle, :dsod_otheroption) ''' CFactory().give_me_db().execute(sql_register_dm_metadata_plugins_clear) plugins_root_dir = CSys.get_plugins_root_dir() plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir) for plugins_type in plugins_type_list: if CFile.is_dir(CFile.join_file( plugins_root_dir, plugins_type)) and ( not (str(plugins_type)).startswith('_')): plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), plugins_type) path = CFile.join_file(CSys.get_plugins_root_dir(), plugins_type) plugins_file_list = CFile.file_or_subpath_of_path( path, '{0}_*.{1}'.format(self.Name_Plugins, self.FileExt_Py)) for file_name_without_path in plugins_file_list: file_main_name = CFile.file_main_name( file_name_without_path) class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, file_main_name, None) plugins_info = class_classified_obj.get_information() json_obj = CJson() json_obj.set_value_of_name( self.Name_Is_Spatial, CUtils.dict_value_by_name( plugins_info, CPlugins.Plugins_Info_Is_Spatial, False)) json_obj.set_value_of_name( self.Name_Is_DataSet, CUtils.dict_value_by_name( plugins_info, CPlugins.Plugins_Info_Is_Dataset, False)) plugins_info['dsod_otheroption'] = json_obj.to_json() print('{0}/{1}:{2}'.format(plugins_type, file_main_name, plugins_info)) CFactory().give_me_db().execute( sql_unregister_dm_metadata_plugins, plugins_info) CFactory().give_me_db().execute( sql_register_dm_metadata_plugins, plugins_info)
def classified(self): """ 设计国土行业数据guoqing_frame的验证规则(国情影像—分幅影像) 完成 负责人 王学谦 在这里检验guoqing_frame的识别规则 :return: """ super().classified() file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 初始化需要的参数 file_path = self.file_info.file_path file_object_name = file_main_name[:] if len(file_main_name) >= 21: # 本类文件默认至少为20位 file_object_name = file_main_name[:20] # 截取前20位 elif len(file_main_name) == 20: # 20位基本为附属文件 pass else: return self.Object_Confirm_IUnKnown, self._object_name match_str = '(?i)^' + file_object_name + r'[a-zA-Z][.]tif' check_file_main_name_exist = \ CFile.find_file_or_subpath_of_path(file_path, match_str, CFile.MatchType_Regex) if not check_file_main_name_exist: # 检查主文件存在性 return self.Object_Confirm_IUnKnown, self._object_name # 文件名第1,4,11,12,16,21位为字母,第2,3,5-10,14,15,17-20位是数字 name_sub_1 = file_main_name[0:1] name_sub_2_to_3 = file_main_name[1:3] name_sub_4 = file_main_name[3:4] name_sub_5_to_10 = file_main_name[4:10] name_sub_11_to_12 = file_main_name[10:12] name_sub_14_to_15 = file_main_name[13:15] name_sub_16 = file_main_name[15:16] name_sub_17_to_20 = file_main_name[16:20] name_sub_21 = file_main_name[20:21] if CUtils.text_is_alpha(name_sub_1) is False \ or CUtils.text_is_numeric(name_sub_2_to_3) is False \ or CUtils.text_is_alpha(name_sub_4) is False \ or CUtils.text_is_numeric(name_sub_5_to_10) is False \ or CUtils.text_is_alpha(name_sub_11_to_12) is False \ or CUtils.text_is_numeric(name_sub_14_to_15) is False \ or CUtils.text_is_alpha(name_sub_16) is False \ or CUtils.text_is_numeric(name_sub_17_to_20) is False \ or CUtils.text_is_alpha(name_sub_21) is False: return self.Object_Confirm_IUnKnown, self._object_name if len(file_main_name) == 21 and CUtils.equal_ignore_case( file_ext, 'tif'): self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name self.add_file_to_detail_list(file_object_name) # 在这里设置不同名的附属文件 else: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name
def metadata_bus_dict_process_custom(self, metadata_bus_dict): super().metadata_bus_dict_process_custom(metadata_bus_dict) centertime = CUtils.dict_value_by_name(metadata_bus_dict, 'centertime', None) publishdate = CUtils.dict_value_by_name(metadata_bus_dict, 'publishdate', None) if not CUtils.equal_ignore_case(centertime, ''): centertime = centertime[0:8] + ' ' + centertime[8:] metadata_bus_dict['centertime'] = centertime if not CUtils.equal_ignore_case(publishdate, ''): publishdate = publishdate[0:8] + ' ' + publishdate[8:] metadata_bus_dict['publishdate'] = publishdate
def classified(self): file_path = self.file_info.file_path file_ext = self.file_info.file_ext if CUtils.text_match_re(file_path, r'(?i)\d{4}.{2}[\\\\/]FenFu[\\\\/]' + self.get_coordinate_system_title()) \ and CUtils.equal_ignore_case(file_ext, 'shp'): self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None return self._object_confirm, self._object_name else: return super().classified()
def __condition_list_2_sql(self, field, value_list, quoted_value): if quoted_value: in_sql = CUtils.list_2_str(value_list, "'", ", ", "'", True) else: in_sql = CUtils.list_2_str(value_list, "", ", ", "", True) if CUtils.equal_ignore_case(in_sql, ''): return "" else: return "{0} in ({1})".format(field, in_sql)
def metadata_bus_dict_process_custom(self, metadata_bus_dict): """ 对部分需要进行运算的数据进行处理 """ super().metadata_bus_dict_process_custom(metadata_bus_dict) productattribute = CUtils.dict_value_by_name(metadata_bus_dict, 'productattribute', None) if not CUtils.equal_ignore_case(productattribute, ''): if len(productattribute) > 2: metadata_bus_dict['productattribute'] = productattribute[:2]