def add_file_to_detail_list(self, match_name): """ 设定国土行业数据国情的附属文件的验证规则(镶嵌影像) 完成 负责人 王学谦 在这里检验国情的附属文件 :return: """ file_main_name = self._object_name file_path = self.file_info.file_path # 正则匹配附属文件 if not CUtils.equal_ignore_case(file_path, ''): match_str = '{0}*.*'.format(match_name) match_file_list = CFile.file_or_dir_fullname_of_path( file_path, False, match_str, CFile.MatchType_Common) match_str_main_name = r'(?i)^{0}[FMP]$'.format(match_name) # 主附属 ext_list = [ 'rar', 'zip', 'doc', 'docx', 'xls', 'xlsx', 'txt', 'xml' ] for file_with_path in match_file_list: if CUtils.equal_ignore_case( CFile.file_main_name(file_with_path), file_main_name): # 去除自身与同名文件 pass elif CUtils.text_match_re(CFile.file_main_name(file_with_path), match_str_main_name): self.add_file_to_details(file_with_path) # 将文件加入到附属文件列表中 elif CFile.file_ext(file_with_path).lower() in ext_list: self.add_file_to_details(file_with_path) else: pass
def __init__(self, file_type, file_name_with_full_path): self.__file_name_with_full_path = file_name_with_full_path self.__file_name_without_path = CFile.file_name( self.file_name_with_full_path) self.__file_main_name = CFile.file_main_name( self.file_name_with_full_path) self.__file_ext = CFile.file_ext(self.file_name_with_full_path) self.__file_path = CFile.file_path(self.file_name_with_full_path) self.__file_main_name_with_full_path = CFile.join_file( self.file_path, self.file_main_name) self.__file_type = file_type self.__file_existed = CFile.file_or_path_exist( self.file_name_with_full_path) if self.__file_existed: if CFile.is_file(self.file_name_with_full_path): self.__file_size = CFile.file_size( self.file_name_with_full_path) self.__file_create_time = CFile.file_create_time( self.file_name_with_full_path) self.__file_access_time = CFile.file_access_time( self.file_name_with_full_path) self.__file_modify_time = CFile.file_modify_time( self.file_name_with_full_path)
def __inbound_object_detail_of_schema(self, list_file_fullname): sql_detail_insert = ''' INSERT INTO dm2_storage_obj_detail( dodid, dodobjectid, dodfilename, dodfileext, dodfilesize, dodfilecreatetime, dodfilemodifytime, dodlastmodifytime, dodfiletype) VALUES ( :dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize, :dodfilecreatetime, :dodfilemodifytime, now(), :dodfiletype) ''' sql_detail_insert_params_list = [] # query_storage_id = self.file_info.storage_id query_file_relation_name = self.file_info.file_name_with_rel_path for item_file_name_with_path in list_file_fullname: CLogger().debug(item_file_name_with_path) if not CFile.file_or_path_exist(item_file_name_with_path): continue params = dict() file_relation_name = CFile.file_relation_path( item_file_name_with_path, self.file_info.root_path) if CUtils.equal_ignore_case(query_file_relation_name, file_relation_name): params['dodid'] = self.object_id else: params['dodid'] = CUtils.one_id() # 文件类型 params['dodfiletype'] = self.FileType_File if CFile.is_dir(item_file_name_with_path): params['dodfiletype'] = self.FileType_Dir params['dodobjectid'] = self.object_id params['dodfilename'] = CFile.unify(file_relation_name) params['dodfileext'] = CFile.file_ext(item_file_name_with_path) params['dodfilesize'] = CFile.file_size(item_file_name_with_path) params['dodfilecreatetime'] = CFile.file_create_time( item_file_name_with_path) params['dodfilemodifytime'] = CFile.file_modify_time( item_file_name_with_path) # params['dodstorageid'] = query_storage_id # params['dodfilerelationname'] = CFile.file_relation_path( # item_file_name_with_path, # self.file_info.root_path) sql_params_tuple = (sql_detail_insert, params) sql_detail_insert_params_list.append(sql_params_tuple) if len(sql_detail_insert_params_list) > 0: try: CFactory().give_me_db( self.file_info.db_server_id).execute_batch( sql_detail_insert_params_list) except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format( error.__str__())) return CResult.merge_result(self.Failure, '处理失败!') return CResult.merge_result(self.Success, '处理完毕!')
def __stat_object_detail_of_schema(self) -> str: """ 将数据附属文件的统计信息入库 . 仅适用于Directory_Itself模式 :return: """ result_sub_dir_count, result_file_count, result_file_size_sum = CFile.stat_of_path( self.__detail_file_path__, self.__detail_file_recurse__, self.__detail_file_match_text__, self.__detail_file_match_type__) query_file_relation_name = self.file_info.file_name_with_rel_path params = dict() file_relation_name = CFile.file_relation_path( self.__detail_file_path__, self.file_info.root_path) if CUtils.equal_ignore_case(query_file_relation_name, file_relation_name): params['dodid'] = self.object_id else: params['dodid'] = CUtils.one_id() params['dodfiletype'] = self.FileType_Dir params['dodfileext'] = None if CFile.is_file(self.__detail_file_path__): params['dodfiletype'] = self.FileType_File params['dodfileext'] = CFile.file_ext(self.__detail_file_path__) params['dodobjectid'] = self.object_id params['dodfilename'] = CFile.unify(file_relation_name) params['doddircount'] = result_sub_dir_count params['dodfilecount'] = result_file_count params['dodfilesize'] = result_file_size_sum params['dodfilecreatetime'] = CFile.file_create_time( self.__detail_file_path__) params['dodfilemodifytime'] = CFile.file_modify_time( self.__detail_file_path__) try: CFactory().give_me_db(self.file_info.db_server_id).execute( ''' INSERT INTO dm2_storage_obj_detail( dodid, dodobjectid, dodfilename, dodfileext, dodfilesize, doddircount, dodfilecount, dodfilecreatetime, dodfilemodifytime, dodlastmodifytime, dodfiletype) VALUES ( :dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize, :doddircount, :dodfilecount, :dodfilecreatetime, :dodfilemodifytime, now(), :dodfiletype) ''', params) return CResult.merge_result(self.Success, '处理完毕!') except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__())) return CResult.merge_result( self.Failure, '数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
def is_raster_file_integrity(cls, raster_file_with_path: str) -> bool: """ 判断影像数据的文件完整性,img xxx.img 文件 栅格数据可读 错误 xxx.ige 文件 img小于1M时必须存在 警告 @param raster_file_with_path: @return: """ file_ext = CFile.file_ext(raster_file_with_path) if file_ext.lower() == 'img': size = CFile.file_size(raster_file_with_path) if size < 1024 * 1024: file_main_name = CFile.file_main_name(raster_file_with_path) file_path = CFile.file_path(raster_file_with_path) ige = CFile.join_file(file_path, file_main_name + '.ige') if not CFile.file_or_path_exist(ige): return False return True
def classified(self): """ 关键字识别 """ super().classified() # 预获取需要的参数 file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name file_ext = self.file_info.file_ext # 预定义逻辑参数 数据文件匹配 object_file_name_flag = False object_file_path_flag = False object_file_ext_flag = False object_affiliated_file_main_flag = False object_file_affiliated_flag = False object_keyword_list = self.get_classified_character_of_object_keyword() if len(object_keyword_list) > 0: for keyword_info in object_keyword_list: keyword_id = CUtils.dict_value_by_name(keyword_info, self.Name_ID, None) regex_match = CUtils.dict_value_by_name( keyword_info, self.Name_RegularExpression, '.*') if regex_match is None: regex_match = '.*' if CUtils.equal_ignore_case(keyword_id, self.Name_FileName): if CUtils.text_match_re(file_main_name, regex_match): object_file_name_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath): if CUtils.text_match_re(file_path, regex_match): object_file_path_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt): if CUtils.text_match_re(file_ext, regex_match): object_file_ext_flag = True else: same_name_file_list = CFile.file_or_dir_fullname_of_path( file_path, False, '(?i)^' + file_main_name + '[.].*$', CFile.MatchType_Regex) if len(same_name_file_list) > 0: for same_name_file in same_name_file_list: same_name_file_ext = CFile.file_ext( same_name_file) if CUtils.text_match_re( same_name_file_ext, regex_match): object_affiliated_file_main_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileAffiliated): affiliated_file_path = CUtils.dict_value_by_name( keyword_info, self.Name_FilePath, None) if affiliated_file_path is not None: if CFile.find_file_or_subpath_of_path( affiliated_file_path, regex_match, CFile.MatchType_Regex): object_file_affiliated_flag = True else: object_file_affiliated_flag = True # 预定义逻辑参数 附属文件匹配 affiliated_file_name_flag = False affiliated_file_path_flag = False affiliated_file_ext_flag = False affiliated_file_main_flag = False affiliated_keyword_list = self.get_classified_character_of_affiliated_keyword( ) if len(affiliated_keyword_list) > 0: for keyword_info in affiliated_keyword_list: keyword_id = CUtils.dict_value_by_name(keyword_info, self.Name_ID, None) regex_match = CUtils.dict_value_by_name( keyword_info, self.Name_RegularExpression, '.*') if regex_match is None: regex_match = '.*' if CUtils.equal_ignore_case(keyword_id, self.Name_FileName): if CUtils.text_match_re(file_main_name, regex_match): affiliated_file_name_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath): if CUtils.text_match_re(file_path, regex_match): affiliated_file_path_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt): if CUtils.text_match_re(file_ext, regex_match): affiliated_file_ext_flag = True elif CUtils.equal_ignore_case(keyword_id, self.Name_FileMain): affiliated_file_path = CUtils.dict_value_by_name( keyword_info, self.Name_FilePath, None) if affiliated_file_path is not None: if CFile.find_file_or_subpath_of_path( affiliated_file_path, regex_match, CFile.MatchType_Regex): affiliated_file_main_flag = True if object_file_name_flag and object_file_path_flag and \ object_file_ext_flag and object_file_affiliated_flag: self._object_confirm = self.Object_Confirm_IKnown self._object_name = file_main_name self.set_custom_affiliated_file() elif affiliated_file_name_flag and affiliated_file_path_flag and \ affiliated_file_ext_flag and affiliated_file_main_flag: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None elif object_file_name_flag and object_file_path_flag and object_affiliated_file_main_flag: self._object_confirm = self.Object_Confirm_IKnown_Not self._object_name = None else: self._object_confirm = self.Object_Confirm_IUnKnown self._object_name = None return self._object_confirm, self._object_name
def image2view(self, target_ds, view_path: str) -> str: """ 影像文件转jpg或png 对波段数据类型为Byte(8位无符号整型)的影像,可采用gdal.Translate()方法直接进行文件格式转换。 对波段数据类型为UInt16(16位无符号整型)、Int16(16位有符号整型)的影像: 由于JPEG不支持16位影像的转换,且PNG转换效果非常不理想,图像轮廓模糊。 因此对16位影像采用百分比截断的方法压缩至0~255的范围,改变了像素深度,降低到8位。 该方法生成的快视图的转换效果与gdal.Translate()方法生成的相比,图像轮廓清晰,可辨识度高。 注:gdal == 3.1.3 :param target_ds: 影像临时文件的数据集 :param view_path: 快视图或拇指图文件地址 :return: """ cols = target_ds.RasterXSize rows = target_ds.RasterYSize band_count = target_ds.RasterCount band = target_ds.GetRasterBand(1) data_type_name = gdal.GetDataTypeName(band.DataType) # 检查影像位深度(波段数据类型) if CUtils.equal_ignore_case(data_type_name, 'Byte'): # 对波段数据类型为Byte(8位无符号整型)的影像进行转换 if CFile.check_and_create_directory(view_path): if CFile.file_ext(view_path).lower() == 'jpg': out = gdal.Translate(view_path, target_ds, format='JPEG') elif CFile.file_ext(view_path).lower() == 'png': out = gdal.Translate(view_path, target_ds, format='PNG') else: # 对波段数据类型为UInt16(16位无符号整型)、Int16(16位有符号整型)的影像进行转换 # 检查影像波段数并读取 if band_count >= 3: bandsOrder = [3, 2, 1] data = np.empty([rows, cols, 3], dtype=float) for i in range(3): band = target_ds.GetRasterBand(bandsOrder[i]) data1 = band.ReadAsArray(0, 0, cols, rows) data[:, :, i] = data1 if CFile.check_and_create_directory(view_path): # 百分比截断压缩影像,将像元取值限定在0~255 lower_percent = 0.6 higher_percent = 99.4 n = data.shape[2] out = np.zeros_like(data, dtype=np.uint8) for i in range(n): a = 0 b = 255 c = np.percentile(data[:, :, i], lower_percent) d = np.percentile(data[:, :, i], higher_percent) t = a + (data[:, :, i] - c) * (b - a) / (d - c) t[t < a] = a t[t > b] = b out[:, :, i] = t outImg = Image.fromarray(np.uint8(out)) outImg.save(view_path) else: data = np.empty([rows, cols], dtype=float) band = target_ds.GetRasterBand(1) data1 = band.ReadAsArray(0, 0, cols, rows) data[:, :] = data1 if CFile.check_and_create_directory(view_path): # 百分比截断压缩影像,将像元取值限定在0~255 lower_percent = 0.6 higher_percent = 99.4 out = np.zeros_like(data, dtype=np.uint8) a = 0 b = 255 c = np.percentile(data[:, :], lower_percent) d = np.percentile(data[:, :], higher_percent) t = a + (data[:, :] - c) * (b - a) / (d - c) t[t < a] = a t[t > b] = b out[:, :] = t outImg = Image.fromarray(np.uint8(out)) outImg.save(view_path) del target_ds del out return view_path
def qa_file_custom(self, parser: CMetaDataParser): """ 自定义的文件存在性质检, 发生在元数据解析之前 完成 负责人 王学谦 :param parser: :return: """ super().qa_file_custom(parser) file_path = self.file_info.file_path affiliated_file_path = '矢量'.join(file_path.rsplit('影像', 1)) # 替换最后一个字符 file_main_name = self.file_info.file_main_name file_object_name_list = re.split('[-_/]', file_main_name, 1) affiliated_file_reg = '(?i)^' + file_object_name_list[0] + '.*[.].*$' shp_file_list = CFile.file_or_subpath_of_path( affiliated_file_path, affiliated_file_reg, CFile.MatchType_Regex ) parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'shp', self.Name_Title: 'shp文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本数据缺少shp文件' } ) parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'dbf', self.Name_Title: '属性数据文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本数据缺少属性数据文件' } ) parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'prj', self.Name_Title: '投影文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本数据缺少投影文件' } ) parser.metadata.quality.append_total_quality( { self.Name_FileName: '', self.Name_ID: 'shx', self.Name_Title: '索引文件', self.Name_Result: self.QA_Result_Error, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '本数据缺少索引文件' } ) for shp_file in shp_file_list: if CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'shp'): parser.metadata.quality.append_total_quality( { self.Name_FileName: shp_file, self.Name_ID: 'shp', self.Name_Title: 'shp文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: 'shp文件[{0}]存在'.format(shp_file) } ) elif CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'dbf'): parser.metadata.quality.append_total_quality( { self.Name_FileName: shp_file, self.Name_ID: 'dbf', self.Name_Title: '属性数据文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '属性数据文件[{0}]存在'.format(shp_file) } ) elif CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'prj'): parser.metadata.quality.append_total_quality( { self.Name_FileName: shp_file, self.Name_ID: 'prj', self.Name_Title: '投影文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '投影文件[{0}]存在'.format(shp_file) } ) elif CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'shx'): parser.metadata.quality.append_total_quality( { self.Name_FileName: shp_file, self.Name_ID: 'shx', self.Name_Title: '索引文件', self.Name_Result: self.QA_Result_Pass, self.Name_Group: self.QA_Group_Data_Integrity, self.Name_Message: '索引文件[{0}]存在'.format(shp_file) } )