def add_file_to_detail_list(self, match_name):
"""
设定国土行业数据国情的附属文件的验证规则（镶嵌影像）
完成 负责人 王学谦 在这里检验国情的附属文件
:return:
"""
file_main_name = self._object_name
file_path = self.file_info.file_path
# 正则匹配附属文件
if not CUtils.equal_ignore_case(file_path, ''):
match_str = '{0}*.*'.format(match_name)
match_file_list = CFile.file_or_dir_fullname_of_path(
file_path, False, match_str, CFile.MatchType_Common)
match_str_main_name = r'(?i)^{0}[FMP]$'.format(match_name) # 主附属
ext_list = [
'rar', 'zip', 'doc', 'docx', 'xls', 'xlsx', 'txt', 'xml'
]
for file_with_path in match_file_list:
if CUtils.equal_ignore_case(
CFile.file_main_name(file_with_path),
file_main_name): # 去除自身与同名文件
pass
elif CUtils.text_match_re(CFile.file_main_name(file_with_path),
match_str_main_name):
self.add_file_to_details(file_with_path) # 将文件加入到附属文件列表中
elif CFile.file_ext(file_with_path).lower() in ext_list:
self.add_file_to_details(file_with_path)
else:
pass
def __init__(self, file_type, file_name_with_full_path):
self.__file_name_with_full_path = file_name_with_full_path
self.__file_name_without_path = CFile.file_name(
self.file_name_with_full_path)
self.__file_main_name = CFile.file_main_name(
self.file_name_with_full_path)
self.__file_ext = CFile.file_ext(self.file_name_with_full_path)
self.__file_path = CFile.file_path(self.file_name_with_full_path)
self.__file_main_name_with_full_path = CFile.join_file(
self.file_path, self.file_main_name)
self.__file_type = file_type
self.__file_existed = CFile.file_or_path_exist(
self.file_name_with_full_path)
if self.__file_existed:
if CFile.is_file(self.file_name_with_full_path):
self.__file_size = CFile.file_size(
self.file_name_with_full_path)
self.__file_create_time = CFile.file_create_time(
self.file_name_with_full_path)
self.__file_access_time = CFile.file_access_time(
self.file_name_with_full_path)
self.__file_modify_time = CFile.file_modify_time(
self.file_name_with_full_path)
def __inbound_object_detail_of_schema(self, list_file_fullname):
sql_detail_insert = '''
INSERT INTO dm2_storage_obj_detail(
dodid, dodobjectid, dodfilename, dodfileext, dodfilesize,
dodfilecreatetime, dodfilemodifytime,
dodlastmodifytime, dodfiletype)
VALUES (
:dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize,
:dodfilecreatetime, :dodfilemodifytime, now(),
:dodfiletype)
'''
sql_detail_insert_params_list = []
# query_storage_id = self.file_info.storage_id
query_file_relation_name = self.file_info.file_name_with_rel_path
for item_file_name_with_path in list_file_fullname:
CLogger().debug(item_file_name_with_path)
if not CFile.file_or_path_exist(item_file_name_with_path):
continue
params = dict()
file_relation_name = CFile.file_relation_path(
item_file_name_with_path, self.file_info.root_path)
if CUtils.equal_ignore_case(query_file_relation_name,
file_relation_name):
params['dodid'] = self.object_id
else:
params['dodid'] = CUtils.one_id()
# 文件类型
params['dodfiletype'] = self.FileType_File
if CFile.is_dir(item_file_name_with_path):
params['dodfiletype'] = self.FileType_Dir
params['dodobjectid'] = self.object_id
params['dodfilename'] = CFile.unify(file_relation_name)
params['dodfileext'] = CFile.file_ext(item_file_name_with_path)
params['dodfilesize'] = CFile.file_size(item_file_name_with_path)
params['dodfilecreatetime'] = CFile.file_create_time(
item_file_name_with_path)
params['dodfilemodifytime'] = CFile.file_modify_time(
item_file_name_with_path)
# params['dodstorageid'] = query_storage_id
# params['dodfilerelationname'] = CFile.file_relation_path(
# item_file_name_with_path,
# self.file_info.root_path)
sql_params_tuple = (sql_detail_insert, params)
sql_detail_insert_params_list.append(sql_params_tuple)
if len(sql_detail_insert_params_list) > 0:
try:
CFactory().give_me_db(
self.file_info.db_server_id).execute_batch(
sql_detail_insert_params_list)
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(
error.__str__()))
return CResult.merge_result(self.Failure, '处理失败!')
return CResult.merge_result(self.Success, '处理完毕!')
def __stat_object_detail_of_schema(self) -> str:
"""
将数据附属文件的统计信息入库
. 仅适用于Directory_Itself模式
:return:
"""
result_sub_dir_count, result_file_count, result_file_size_sum = CFile.stat_of_path(
self.__detail_file_path__, self.__detail_file_recurse__,
self.__detail_file_match_text__, self.__detail_file_match_type__)
query_file_relation_name = self.file_info.file_name_with_rel_path
params = dict()
file_relation_name = CFile.file_relation_path(
self.__detail_file_path__, self.file_info.root_path)
if CUtils.equal_ignore_case(query_file_relation_name,
file_relation_name):
params['dodid'] = self.object_id
else:
params['dodid'] = CUtils.one_id()
params['dodfiletype'] = self.FileType_Dir
params['dodfileext'] = None
if CFile.is_file(self.__detail_file_path__):
params['dodfiletype'] = self.FileType_File
params['dodfileext'] = CFile.file_ext(self.__detail_file_path__)
params['dodobjectid'] = self.object_id
params['dodfilename'] = CFile.unify(file_relation_name)
params['doddircount'] = result_sub_dir_count
params['dodfilecount'] = result_file_count
params['dodfilesize'] = result_file_size_sum
params['dodfilecreatetime'] = CFile.file_create_time(
self.__detail_file_path__)
params['dodfilemodifytime'] = CFile.file_modify_time(
self.__detail_file_path__)
try:
CFactory().give_me_db(self.file_info.db_server_id).execute(
'''
INSERT INTO dm2_storage_obj_detail(
dodid, dodobjectid, dodfilename, dodfileext, dodfilesize, doddircount, dodfilecount,
dodfilecreatetime, dodfilemodifytime, dodlastmodifytime, dodfiletype)
VALUES (
:dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize, :doddircount, :dodfilecount,
:dodfilecreatetime, :dodfilemodifytime, now(), :dodfiletype)
''', params)
return CResult.merge_result(self.Success, '处理完毕!')
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
return CResult.merge_result(
self.Failure, '数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
def is_raster_file_integrity(cls, raster_file_with_path: str) -> bool:
"""
判断影像数据的文件完整性，img
xxx.img 文件 栅格数据可读 错误
xxx.ige 文件 img小于1M时必须存在 警告
@param raster_file_with_path:
@return:
"""
file_ext = CFile.file_ext(raster_file_with_path)
if file_ext.lower() == 'img':
size = CFile.file_size(raster_file_with_path)
if size < 1024 * 1024:
file_main_name = CFile.file_main_name(raster_file_with_path)
file_path = CFile.file_path(raster_file_with_path)
ige = CFile.join_file(file_path, file_main_name + '.ige')
if not CFile.file_or_path_exist(ige):
return False
return True
def classified(self):
"""
关键字识别
"""
super().classified()
# 预获取需要的参数
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
# 预定义逻辑参数 数据文件匹配
object_file_name_flag = False
object_file_path_flag = False
object_file_ext_flag = False
object_affiliated_file_main_flag = False
object_file_affiliated_flag = False
object_keyword_list = self.get_classified_character_of_object_keyword()
if len(object_keyword_list) > 0:
for keyword_info in object_keyword_list:
keyword_id = CUtils.dict_value_by_name(keyword_info,
self.Name_ID, None)
regex_match = CUtils.dict_value_by_name(
keyword_info, self.Name_RegularExpression, '.*')
if regex_match is None:
regex_match = '.*'
if CUtils.equal_ignore_case(keyword_id, self.Name_FileName):
if CUtils.text_match_re(file_main_name, regex_match):
object_file_name_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath):
if CUtils.text_match_re(file_path, regex_match):
object_file_path_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt):
if CUtils.text_match_re(file_ext, regex_match):
object_file_ext_flag = True
else:
same_name_file_list = CFile.file_or_dir_fullname_of_path(
file_path, False,
'(?i)^' + file_main_name + '[.].*$',
CFile.MatchType_Regex)
if len(same_name_file_list) > 0:
for same_name_file in same_name_file_list:
same_name_file_ext = CFile.file_ext(
same_name_file)
if CUtils.text_match_re(
same_name_file_ext, regex_match):
object_affiliated_file_main_flag = True
elif CUtils.equal_ignore_case(keyword_id,
self.Name_FileAffiliated):
affiliated_file_path = CUtils.dict_value_by_name(
keyword_info, self.Name_FilePath, None)
if affiliated_file_path is not None:
if CFile.find_file_or_subpath_of_path(
affiliated_file_path, regex_match,
CFile.MatchType_Regex):
object_file_affiliated_flag = True
else:
object_file_affiliated_flag = True
# 预定义逻辑参数 附属文件匹配
affiliated_file_name_flag = False
affiliated_file_path_flag = False
affiliated_file_ext_flag = False
affiliated_file_main_flag = False
affiliated_keyword_list = self.get_classified_character_of_affiliated_keyword(
)
if len(affiliated_keyword_list) > 0:
for keyword_info in affiliated_keyword_list:
keyword_id = CUtils.dict_value_by_name(keyword_info,
self.Name_ID, None)
regex_match = CUtils.dict_value_by_name(
keyword_info, self.Name_RegularExpression, '.*')
if regex_match is None:
regex_match = '.*'
if CUtils.equal_ignore_case(keyword_id, self.Name_FileName):
if CUtils.text_match_re(file_main_name, regex_match):
affiliated_file_name_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath):
if CUtils.text_match_re(file_path, regex_match):
affiliated_file_path_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt):
if CUtils.text_match_re(file_ext, regex_match):
affiliated_file_ext_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileMain):
affiliated_file_path = CUtils.dict_value_by_name(
keyword_info, self.Name_FilePath, None)
if affiliated_file_path is not None:
if CFile.find_file_or_subpath_of_path(
affiliated_file_path, regex_match,
CFile.MatchType_Regex):
affiliated_file_main_flag = True
if object_file_name_flag and object_file_path_flag and \
object_file_ext_flag and object_file_affiliated_flag:
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
self.set_custom_affiliated_file()
elif affiliated_file_name_flag and affiliated_file_path_flag and \
affiliated_file_ext_flag and affiliated_file_main_flag:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
elif object_file_name_flag and object_file_path_flag and object_affiliated_file_main_flag:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
else:
self._object_confirm = self.Object_Confirm_IUnKnown
self._object_name = None
return self._object_confirm, self._object_name
def image2view(self, target_ds, view_path: str) -> str:
"""
影像文件转jpg或png
对波段数据类型为Byte（8位无符号整型）的影像，可采用gdal.Translate()方法直接进行文件格式转换。
对波段数据类型为UInt16（16位无符号整型）、Int16（16位有符号整型）的影像：
由于JPEG不支持16位影像的转换，且PNG转换效果非常不理想，图像轮廓模糊。
因此对16位影像采用百分比截断的方法压缩至0~255的范围，改变了像素深度，降低到8位。
该方法生成的快视图的转换效果与gdal.Translate()方法生成的相比，图像轮廓清晰，可辨识度高。
注：gdal == 3.1.3
:param target_ds: 影像临时文件的数据集
:param view_path: 快视图或拇指图文件地址
:return:
"""
cols = target_ds.RasterXSize
rows = target_ds.RasterYSize
band_count = target_ds.RasterCount
band = target_ds.GetRasterBand(1)
data_type_name = gdal.GetDataTypeName(band.DataType)
# 检查影像位深度（波段数据类型）
if CUtils.equal_ignore_case(data_type_name, 'Byte'):
# 对波段数据类型为Byte（8位无符号整型）的影像进行转换
if CFile.check_and_create_directory(view_path):
if CFile.file_ext(view_path).lower() == 'jpg':
out = gdal.Translate(view_path, target_ds, format='JPEG')
elif CFile.file_ext(view_path).lower() == 'png':
out = gdal.Translate(view_path, target_ds, format='PNG')
else:
# 对波段数据类型为UInt16（16位无符号整型）、Int16（16位有符号整型）的影像进行转换
# 检查影像波段数并读取
if band_count >= 3:
bandsOrder = [3, 2, 1]
data = np.empty([rows, cols, 3], dtype=float)
for i in range(3):
band = target_ds.GetRasterBand(bandsOrder[i])
data1 = band.ReadAsArray(0, 0, cols, rows)
data[:, :, i] = data1
if CFile.check_and_create_directory(view_path):
# 百分比截断压缩影像，将像元取值限定在0~255
lower_percent = 0.6
higher_percent = 99.4
n = data.shape[2]
out = np.zeros_like(data, dtype=np.uint8)
for i in range(n):
a = 0
b = 255
c = np.percentile(data[:, :, i], lower_percent)
d = np.percentile(data[:, :, i], higher_percent)
t = a + (data[:, :, i] - c) * (b - a) / (d - c)
t[t < a] = a
t[t > b] = b
out[:, :, i] = t
outImg = Image.fromarray(np.uint8(out))
outImg.save(view_path)
else:
data = np.empty([rows, cols], dtype=float)
band = target_ds.GetRasterBand(1)
data1 = band.ReadAsArray(0, 0, cols, rows)
data[:, :] = data1
if CFile.check_and_create_directory(view_path):
# 百分比截断压缩影像，将像元取值限定在0~255
lower_percent = 0.6
higher_percent = 99.4
out = np.zeros_like(data, dtype=np.uint8)
a = 0
b = 255
c = np.percentile(data[:, :], lower_percent)
d = np.percentile(data[:, :], higher_percent)
t = a + (data[:, :] - c) * (b - a) / (d - c)
t[t < a] = a
t[t > b] = b
out[:, :] = t
outImg = Image.fromarray(np.uint8(out))
outImg.save(view_path)
del target_ds
del out
return view_path
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 王学谦
:param parser:
:return:
"""
super().qa_file_custom(parser)
file_path = self.file_info.file_path
affiliated_file_path = '矢量'.join(file_path.rsplit('影像', 1)) # 替换最后一个字符
file_main_name = self.file_info.file_main_name
file_object_name_list = re.split('[-_/]', file_main_name, 1)
affiliated_file_reg = '(?i)^' + file_object_name_list[0] + '.*[.].*$'
shp_file_list = CFile.file_or_subpath_of_path(
affiliated_file_path, affiliated_file_reg, CFile.MatchType_Regex
)
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'shp',
self.Name_Title: 'shp文件',
self.Name_Result: self.QA_Result_Error,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本数据缺少shp文件'
}
)
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'dbf',
self.Name_Title: '属性数据文件',
self.Name_Result: self.QA_Result_Error,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本数据缺少属性数据文件'
}
)
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'prj',
self.Name_Title: '投影文件',
self.Name_Result: self.QA_Result_Error,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本数据缺少投影文件'
}
)
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'shx',
self.Name_Title: '索引文件',
self.Name_Result: self.QA_Result_Error,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本数据缺少索引文件'
}
)
for shp_file in shp_file_list:
if CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'shp'):
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: shp_file,
self.Name_ID: 'shp',
self.Name_Title: 'shp文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: 'shp文件[{0}]存在'.format(shp_file)
}
)
elif CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'dbf'):
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: shp_file,
self.Name_ID: 'dbf',
self.Name_Title: '属性数据文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '属性数据文件[{0}]存在'.format(shp_file)
}
)
elif CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'prj'):
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: shp_file,
self.Name_ID: 'prj',
self.Name_Title: '投影文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '投影文件[{0}]存在'.format(shp_file)
}
)
elif CUtils.equal_ignore_case(CFile.file_ext(shp_file), 'shx'):
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: shp_file,
self.Name_ID: 'shx',
self.Name_Title: '索引文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '索引文件[{0}]存在'.format(shp_file)
}
)