def get_sync_dict_list(self, insert_or_updata) -> list:
"""
insert_or_updata 中 self.DB_True为insert，DB_False为updata
本方法的写法为强规则，调用add_value_to_sync_dict_list配置
第一个参数为list，第二个参数为字段名，第三个参数为字段值，第四个参数为特殊配置
"""
object_name = self._obj_name
dsometadataxml_bus = self._dataset.value_by_name(
0, 'dsometadataxml_bus', '')
xml = CXml()
xml.load_xml(dsometadataxml_bus) # 初始化xml
dsometadataxml_bus_type = '{0}'.format(
xml.xpath_one("/root/@type")) # 查询业务元数据类别
if object_name is not None:
if dsometadataxml_bus_type is not None:
if CUtils.equal_ignore_case(dsometadataxml_bus_type, 'mdb'):
return self.get_sync_mdb_dict_list(insert_or_updata)
elif CUtils.equal_ignore_case(dsometadataxml_bus_type, 'mat'):
return self.get_sync_mat_dict_list(insert_or_updata)
elif CUtils.equal_ignore_case(dsometadataxml_bus_type, 'xls') \
or CUtils.equal_ignore_case(dsometadataxml_bus_type, 'xlsx'):
return self.get_sync_xls_dict_list(insert_or_updata)
raise Exception("数据{0}业务元数据类型为'{1}'出现错误，没有可识别的类型".format(
object_name, dsometadataxml_bus_type))
def _layer_is_dataset(self, layer_obj) -> bool:
"""
判断一个层是否是合法的数据集
:param layer_obj:
:return:
"""
driver = self._data_obj.GetDriver()
if driver is None:
return True
layer_name = CUtils.any_2_str(layer_obj.GetName())
if CUtils.equal_ignore_case(driver.name, 'OpenFileGDB'):
if layer_name.upper().startswith('T_1_'):
return False
elif CUtils.equal_ignore_case(driver.name, 'PGeo'):
if layer_name.upper().endswith('_SHAPE_INDEX'):
return False
elif CUtils.equal_ignore_case(layer_name, 'Selections'):
return False
elif CUtils.equal_ignore_case(layer_name, 'SelectedObjects'):
return False
return True
def classified(self):
"""
设计国土行业数据custom的验证规则（自定义影像）
完成 负责人 李宪 在这里检验custom的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
file_object_name = file_main_name
file_name_with_full_path = self.file_info.file_name_with_full_path # 初始化需要的参数
if file_name_with_full_path.endswith('_21at.xml'):
file_object_name = file_main_name[:-5]
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_object_name)
check_file_main_name_exist_tif = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
check_file_main_name_exist_img = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Img))
if (not check_file_main_name_exist_tif) and (
not check_file_main_name_exist_img):
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \
or CUtils.equal_ignore_case(file_ext, self.Name_Img):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = self.file_info.file_main_name
file_detail_xml = '{0}_21at.xml'.format(
self.file_info.file_main_name_with_full_path)
self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def metadata_bus_dict_process_custom(self, metadata_bus_dict):
"""
对部分需要进行运算的数据进行处理
"""
super().metadata_bus_dict_process_custom(metadata_bus_dict)
resolution = CUtils.dict_value_by_name(metadata_bus_dict, 'resolution',
None)
if not CUtils.equal_ignore_case(resolution, ''):
resolution_list = re.split(r'[/]|\s+', resolution.strip())
if len(resolution_list) > 0:
temp_resolution_list = []
for temp_resolution in resolution_list:
temp_resolution_list.append(
CUtils.to_decimal(temp_resolution))
metadata_bus_dict['resolution'] = min(temp_resolution_list)
def classified(self):
"""
设计国土行业数据的dom-10验证规则
完成 负责人 李宪 在这里检验dom-10的元数据文件格式时, 应该一个一个类型的对比, 找到文件时, 将该文件的格式和文件名存储到类的私有属性中, 以便在元数据处理时直接使用
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
check_file_main_name_length = len(file_main_name) == 10
if not check_file_main_name_length:
return self.Object_Confirm_IUnKnown, self._object_name
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_main_name)
check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format(
file_main_name_with_path, self.Name_Tif))
if not check_file_main_name_exist:
return self.Object_Confirm_IUnKnown, self._object_name
"""
下面判别第1位是字母
下面判别第4位是字母
下面判别第23位是数字
下面判别第567位是数字
下面判别第8910位是数字
"""
char_1 = file_main_name[0:1]
char_2_3 = file_main_name[1:3]
char_4 = file_main_name[3:4]
char_5_to_7 = file_main_name[4:7]
char_8_to_10 = file_main_name[7:10]
if CUtils.text_is_alpha(char_1) is False \
or CUtils.text_is_numeric(char_2_3) is False \
or CUtils.text_is_alpha(char_4) is False \
or CUtils.text_is_numeric(char_5_to_7) is False \
or CUtils.text_is_numeric(char_8_to_10) is False:
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def add_file_to_detail_list(self):
"""
设定国土行业数据mosaic的附属文件的验证规则（镶嵌影像）
完成 负责人 李宪 在这里检验mosaic的附属文件的识别规则
:return:
"""
file_detail_xml = '{0}_21at.xml'.format(
self.file_info.file_main_name_with_full_path)
self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中
file_detail_xq = '{0}xq.*'.format(self.file_info.file_main_name)
if not CUtils.equal_ignore_case(self.file_info.file_path, ''):
list_file_fullname_xq = CFile.file_or_dir_fullname_of_path(
self.file_info.file_path, False, file_detail_xq,
CFile.MatchType_Common) # 模糊匹配文件列表
for list_file_fullname in list_file_fullname_xq:
self.add_file_to_details(list_file_fullname) # 将文件加入到附属文件列表中
def db_access_check(self, access_wait_flag, access_forbid_flag, message):
temporary_dict = dict()
temporary_dict['dso_time'] = self._dataset.value_by_name(
0, 'dso_time', '')
temporary_dict['dso_browser'] = self._dataset.value_by_name(
0, 'dso_browser', '')
temporary_dict['dso_thumb'] = self._dataset.value_by_name(
0, 'dso_thumb', '')
temporary_dict['dso_geo_wgs84'] = self._dataset.value_by_name(
0, 'dso_geo_wgs84', '')
for key, value in temporary_dict.items():
if CUtils.equal_ignore_case(value, ''):
message = message + '[数据{0}入库异常!请进行检查与修正！]'.format(
key.replace('dso_', ''))
access_forbid_flag = self.DB_True
return access_wait_flag, access_forbid_flag, message
def __init__(self, file_type, file_name_with_full_path, root_path,
rule_content):
super().__init__(file_type, file_name_with_full_path)
self.__root_path = root_path
if CUtils.equal_ignore_case(root_path, file_name_with_full_path):
self.__file_main_name_with_rel_path = ''
self.__file_name_with_rel_path = ''
self.__file_path_with_rel_path = ''
else:
self.__file_main_name_with_rel_path = CFile.file_relation_path(
self.file_main_name_with_full_path, self.__root_path)
self.__file_name_with_rel_path = CFile.file_relation_path(
self.file_name_with_full_path, self.__root_path)
self.__file_path_with_rel_path = CFile.file_relation_path(
self.file_path, self.__root_path)
self.__rule_content = rule_content
def give_me_db(self, db_id=None) -> CDataBase:
if db_id is None:
rt_db_id = self.DB_Server_ID_Default
else:
rt_db_id = CUtils.any_2_str(db_id)
if CUtils.equal_ignore_case(rt_db_id, ''):
rt_db_id = self.DB_Server_ID_Default
databases = settings.application.xpath_one(self.Name_DataBases, None)
if databases is None:
raise Exception('系统未配置数据库定义, 请检查修正后重试！')
for database in databases:
if rt_db_id == CUtils.dict_value_by_name(database, self.Name_ID, self.DB_Server_ID_Default):
return self.create_db(database)
raise Exception('未找到标示为[{0}]]的数据库定义！'.format(rt_db_id))
def get_ib_schema(self, dataset_type, ib_option):
"""
根据数据集类型, 获取系统设置中的最佳入库模式
:param ib_option: 其他入库的特殊要求
:param dataset_type:
:return:
"""
ib_schema_list = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_Schema_Special, [])
ib_schema_default = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_Schema_Default, None)
for ib_schema in ib_schema_list:
ib_schema_id = CUtils.dict_value_by_name(ib_schema, self.Name_ID,
'')
if CUtils.equal_ignore_case(ib_schema_id, dataset_type):
return ib_schema
else:
return ib_schema_default
def seq_next_value(self, seq_type: int) -> str:
if seq_type == self.Seq_Type_Date_AutoInc:
sql_last_seq_date = '''
select coalesce(gcfgvalue, current_date::text) as last_date, current_date::text as curr_date from ro_global_config where
gcfgcode = 'sys_seq_date_autoinc'
'''
ds_last_seq_date = self.one_row(sql_last_seq_date)
if ds_last_seq_date.is_empty():
self.execute('''
insert into ro_global_config(gcfgid, gcfgcode, gcfgtitle, gcfgvalue, gcfgmemo)
values (10001, 'sys_seq_date_autoinc', '日期自增序列最后记录', current_date::text, null)
''')
ds_last_seq_date = self.one_row(sql_last_seq_date)
last_seq_date = ds_last_seq_date.value_by_name(
0, 'last_date', None)
curr_seq_date = ds_last_seq_date.value_by_name(
0, 'curr_date', None)
if last_seq_date is None:
self.execute('''
update ro_global_config set gcfgvalue = current_date::text
where gcfgcode = 'sys_seq_date_autoinc'
''')
elif not CUtils.equal_ignore_case(last_seq_date, curr_seq_date):
self.execute('''
update ro_global_config set gcfgvalue = current_date::text
where gcfgcode = 'sys_seq_date_autoinc'
''')
next_value = self.one_row(
"select setval('sys_seq_date_autoinc', 1)").value_by_index(
0, 0, 1)
return '{0}-{1}'.format(curr_seq_date,
CUtils.int_2_format_str(next_value, 3))
next_value = self.one_row(
"select nextval('sys_seq_date_autoinc')").value_by_index(
0, 0, 1)
return '{0}-{1}'.format(curr_seq_date,
CUtils.int_2_format_str(next_value, 3))
else:
return CUtils.any_2_str(
self.one_row(
"select nextval('sys_seq_autoinc')").value_by_index(
0, 0, 1))
def create_job_instance(cls, package_directory, package_root_name, package_subdir, package_name, *args, **kwargs):
package_root = os.path.join(package_directory, package_subdir)
dir_list = os.listdir(package_root)
for cur_file in dir_list:
if not CUtils.equal_ignore_case(CFile.file_main_name(cur_file), package_name):
continue
path = os.path.join(package_root, cur_file)
if os.path.isfile(path) and (not cur_file.startswith('__init__')):
package_root_name = '{0}.{1}.{2}'.format(package_root_name, package_subdir, package_name)
package_obj = importlib.import_module(package_root_name)
class_meta = getattr(package_obj, package_name)
class_meta_one = class_meta
obj_id = args[0]
obj_params = args[1]
obj = class_meta_one(obj_id, obj_params)
return obj
else:
raise BusinessNotExistException(package_subdir, package_name)
def db_check_and_update(self, ib_id):
"""
检查并更新dm2_storage_file表中记录
:return:
"""
if not self.ds_file_or_path.is_empty():
# 如果记录已经存在
db_file_modify_time = self.ds_file_or_path.value_by_name(0, 'dsddirlastmodifytime', '')
db_file_size = self.ds_file_or_path.value_by_name(0, 'dsffilesize', 0)
if CUtils.equal_ignore_case(CUtils.any_2_str(db_file_modify_time),
CUtils.any_2_str(self.file_modify_time)) and (
db_file_size == self.file_size):
CLogger().info('文件[{0}]的大小和最后修改时间, 和库中登记的没有变化, 文件将被设置为忽略刷新! '.format(self.file_name_with_full_path))
CFactory().give_me_db(self.db_server_id).execute(
'''
update dm2_storage_file
set dsfScanStatus = 0, dsffilevalid = -1, dsf_ib_id = :ib_id, dsfscanmemo = :message
where dsfid = :dsfid
''',
{
'dsfid': self.my_id,
'ib_id': ib_id,
'message': '文件[{0}]的大小和最后修改时间, 和库中登记的没有变化, 文件将被设置为忽略刷新! '.format(self.file_name_with_full_path)
}
)
else:
CLogger().info('文件[{0}]的大小和最后修改时间, 和库中登记的有变化, 文件将被设置为重新刷新! '.format(self.file_name_with_full_path))
CFactory().give_me_db(self.db_server_id).execute(
'''
update dm2_storage_file
set dsfScanStatus = 1, dsffilevalid = -1, dsf_ib_id = :ib_id, dsfscanmemo = :message
where dsfid = :dsfid
''',
{
'dsfid': self.my_id,
'ib_id': ib_id,
'message': '文件[{0}]的大小和最后修改时间, 和库中登记的有变化, 文件将被设置为重新刷新! '.format(self.file_name_with_full_path)
}
)
else:
CLogger().info('文件[{0}]未在库中登记, 系统将登记该记录! '.format(self.file_name_with_full_path))
self.__db_insert(ib_id)
def get_mdb_connect(self, file_metadata_name_with_path):
"""
:return:
"""
try:
# win下需要安装AccessDatabaseEngine_X64.exe驱动
# linux下需要安装mdbtools与unixODBC-2.3.9.tar与gdal，gdal拥有pgeo驱动
# 并将odbcinst.ini文件的设置一个连接名为Driver=Microsoft Access Driver (*.mdb)的项目
mdb = 'Driver=Microsoft Access Driver (*.mdb);' + 'DBQ={0}'.format(
file_metadata_name_with_path
) # win驱动，安装AccessDatabaseEngine_X64.exe驱动
# get_os_name方法返回值, posix, nt, java对应linux / windows / java虚拟机
if CUtils.equal_ignore_case(CSys.get_os_name(), 'nt'): # win下返回nt
conn = pypyodbc.win_connect_mdb(
mdb) # 安装pypyodbc插件，本插件为python写的，可全平台
else:
conn = pypyodbc.connect(mdb)
except Exception as error:
raise Exception('mdb解析驱动异常:' + error.__str__())
return conn
def classified(self):
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext # 初始化需要的参数
file_object_name = file_main_name[:]
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_object_name)
if CUtils.equal_ignore_case(file_ext, self.Name_Shp):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
if CFile.file_or_path_exist('{0}.{1}'.format(
file_main_name_with_path, self.Name_Shp)):
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
else:
self._object_confirm = self.Object_Confirm_IUnKnown
self._object_name = None
return self._object_confirm, self._object_name
def process(self) -> str:
"""
:return:
"""
super().process()
file_metadata_name_with_path = self.transformer_src_filename
try:
xml_obj = None
if CUtils.equal_ignore_case(self.transformer_type, self.Transformer_DOM_MDB):
xml_obj = self.mdb_to_xml(file_metadata_name_with_path)
else:
raise
if xml_obj is not None:
super().metadata.set_metadata_bus(
self.Success,
'元数据文件[{0}]成功加载! '.format(file_metadata_name_with_path),
self.MetaDataFormat_XML,
xml_obj.to_xml()
)
return CResult.merge_result(
self.Success,
'元数据文件[{0}]成功加载! '.format(file_metadata_name_with_path)
)
else:
raise
except Exception as error:
super().metadata.set_metadata_bus(
self.Exception,
'元数据文件[{0}]格式不合法, 无法处理! 错误原因为{1}'
.format(file_metadata_name_with_path, error.__str__()),
self.MetaDataFormat_Text,
''
)
return CResult.merge_result(
self.Exception,
'元数据文件[{0}]格式不合法, 无法处理! 错误原因为{1}'
.format(file_metadata_name_with_path, error.__str__())
)
def sync(self, object_access, obj_id, obj_name, obj_type, quality) -> str:
"""
处理数管中识别的对象, 与第三方模块的同步
. 如果第三方模块自行处理, 则无需继承本方法
. 如果第三方模块可以处理, 则在本模块中, 从数据库中提取对象的信息, 写入第三方模块的数据表中, 或者调用第三方模块接口
注意: 在本方法中, 不要用_quality_info属性, 因为外部调用方考虑的效率因素, 没有传入!!!
:return:
"""
# 根据objecttype类型查找distribution文件夹下对应的类文件（识别通过objecttype找object_def表中的dsodtype字段与类对象中的info[self.Name_Type]值相同）
if CUtils.equal_ignore_case(self.DataAccess_Pass, object_access):
distribution_obj_real, result = self.__find_module_obj(obj_id)
if not CResult.result_success(result):
return result
elif distribution_obj_real is None:
message = '没有对应的算法, 直接通过!'
result = CResult.merge_result(self.Success, message)
return result
result = distribution_obj_real.sync()
return result
else:
# todo(赵宇飞) 这里如果权限是等待审批或禁用, 则从ap3系列表中删除记录
return CResult.merge_result(self.Success,
'这里需要从ap3系列表中删除记录, 等待实现!')
def process_vector(self) -> str:
try:
# file_name_with_full_path = r'D:\test\vector_test\石嘴山市-3xq.json'
# file_main_name = CFile.file_main_name(file_name_with_full_path)
# file_path = CFile.file_path(file_name_with_full_path)
xml_obj = self.metadata.metadata_xml()
# <editor-fold desc="1.空间坐标信息">
wkt_info = 'POLYGON((${min_x} ${max_y},${max_x} ${max_y},' \
'${max_x} ${min_y},${min_x} ${min_y},${min_x} ${max_y}))'
# 四至坐标
native_max_x = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MaxLon'))
native_max_y = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MaxLat'))
native_min_x = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MinLon'))
native_min_y = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MinLat'))
if (native_max_x is None) or (native_max_y is None) \
or (native_min_x is None) or (native_min_y is None):
native_center_wkt = None
native_bbox_wkt = None
geom_native_wkt = None
else:
dict_native = {
'max_x': CUtils.any_2_str(native_max_x),
'max_y': CUtils.any_2_str(native_max_y),
'min_x': CUtils.any_2_str(native_min_x),
'min_y': CUtils.any_2_str(native_min_y)
}
# 中心坐标
center_x = (native_max_x - native_min_x) / 2 + native_min_x
center_y = (native_max_y - native_min_y) / 2 + native_min_y
native_center_wkt = 'POINT({0} {1})'.format(center_x, center_y)
# 外边框、外包框
native_bbox_wkt = CUtils.replace_placeholder(
wkt_info[:], dict_native)
geom_native_wkt = native_bbox_wkt
file_path = self.file_content.work_root_dir
file_main_name = self.object_name
native_center_filepath = CFile.join_file(
file_path, file_main_name + '_native_center.wkt')
CFile.str_2_file(native_center_wkt, native_center_filepath)
native_bbox_filepath = CFile.join_file(
file_path, file_main_name + '_native_bbox.wkt')
CFile.str_2_file(native_bbox_wkt, native_bbox_filepath)
native_geom_filepath = CFile.join_file(
file_path, file_main_name + '_native_geom.wkt')
CFile.str_2_file(geom_native_wkt, native_geom_filepath)
projection = xml_obj.get_element_text_by_xpath_one(
'/TileMetadata/SpatialReference/PRJ')
if (projection is not None) and (not CUtils.equal_ignore_case(
projection, '')):
source_projection = osr.SpatialReference(wkt=projection)
source = source_projection.GetAttrValue('GEOGCS', 0) # 坐标系名称
prosrs = osr.SpatialReference()
if prosrs.ImportFromWkt(projection) == gdal.CE_None:
proj_wkt = prosrs.ExportToWkt()
native_wkt = proj_wkt
else:
native_wkt = projection
native_proj4 = prosrs.ExportToProj4()
spatial = None
rb = (0, 0)
lu = (0, 0)
geosrs = prosrs.CloneGeogCS()
ct = osr.CreateCoordinateTransformation(prosrs, geosrs)
if ct is not None:
rb = ct.TransformPoint(native_max_x, native_max_y)
lu = ct.TransformPoint(native_min_x, native_min_y)
wgs84_min_x = lu[0]
wgs84_max_y = lu[1]
wgs84_max_x = rb[0]
wgs84_min_y = rb[1]
dict_wgs84 = {
'max_x': CUtils.any_2_str(wgs84_max_x),
'max_y': CUtils.any_2_str(wgs84_max_y),
'min_x': CUtils.any_2_str(wgs84_min_x),
'min_y': CUtils.any_2_str(wgs84_min_y)
}
# 中心坐标
center_x = (wgs84_max_x - wgs84_min_x) / 2 + wgs84_min_x
center_y = (wgs84_max_y - wgs84_min_y) / 2 + wgs84_min_y
wgs84_center_wkt = 'POINT({0} {1})'.format(
center_x, center_y)
# 外边框、外包框
wgs84_bbox_wkt = CUtils.replace_placeholder(
wkt_info[:], dict_wgs84)
wgs84_geom_wkt = wgs84_bbox_wkt
wgs84_center_filepath = CFile.join_file(
file_path, file_main_name + '_wgs84_center.wkt')
CFile.str_2_file(wgs84_center_wkt, wgs84_center_filepath)
wgs84_bbox_filepath = CFile.join_file(
file_path, file_main_name + '_wgs84_bbox.wkt')
CFile.str_2_file(wgs84_bbox_wkt, wgs84_bbox_filepath)
wgs84_geom_filepath = CFile.join_file(
file_path, file_main_name + '_wgs84_geom.wkt')
CFile.str_2_file(wgs84_geom_wkt, wgs84_geom_filepath)
# </editor-fold>
# <editor-fold desc="2.投影信息">
native_source = CResource.Prj_Source_Data
# 坐标系
native_coordinate = None
# 3度带/6度带
native_degree = None
# 投影方式
native_project = None
# 带号
native_zone = None
# 创建SpatialReference对象，导入wkt信息
spatial_ref = osr.SpatialReference(wkt=native_wkt)
if spatial_ref.IsProjected():
native_project = spatial_ref.GetAttrValue('PROJECTION')
native_coordinate = spatial_ref.GetAttrValue('GEOGCS')
# native_degree = spatial_ref.GetAttrValue('GEOGCS|UNIT', 1)
native_degree, native_zone = self.get_prj_degree_zone(
spatial_ref)
elif spatial_ref.IsGeocentric():
native_project = None
native_coordinate = spatial_ref.GetAttrValue('GEOGCS')
native_degree = None
native_zone = None
# </editor-fold>
result = CResult.merge_result(self.Success, '处理完毕!')
result = CResult.merge_result_info(result, self.Name_Prj_Wkt,
native_wkt)
result = CResult.merge_result_info(result, self.Name_Prj_Proj4,
native_proj4)
result = CResult.merge_result_info(result,
self.Name_Prj_Project,
native_project)
result = CResult.merge_result_info(result,
self.Name_Prj_Coordinate,
native_coordinate)
result = CResult.merge_result_info(result,
self.Name_Prj_Source,
native_source)
result = CResult.merge_result_info(result, self.Name_Prj_Zone,
native_zone)
result = CResult.merge_result_info(result,
self.Name_Prj_Degree,
native_degree)
return result
# return CResult.merge_result(self.Success, '处理完毕!')
else:
return CResult.merge_result(self.Success, '处理完毕!')
except Exception as error:
CLogger().warning('矢量数据的空间信息处理出现异常, 错误信息为: {0}'.format(
error.__str__()))
return CResult.merge_result(
self.Failure,
'矢量数据的空间信息处理出现异常,错误信息为：{0}!'.format(error.__str__()))
def process_mission(self, dataset) -> str:
ds_subpath = dataset.value_by_name(0, 'query_subpath', '')
ds_root_path = dataset.value_by_name(0, 'query_root_path', '')
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_id = dataset.value_by_name(0, 'query_dir_id', '')
owner_obj_id = dataset.value_by_name(0, 'query_dir_parent_objid', '')
parent_id = dataset.value_by_name(0, 'query_dir_parent_id', '')
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure,
'{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo,
ds_retry_times
)
)
self.update_dir_status(ds_id, process_result, self.ProcStatus_Error)
return process_result
if CUtils.equal_ignore_case(ds_subpath, ''):
result = CResult.merge_result(CResult.Success, '根目录[{0}]不支持识别为对象, 当前流程被忽略!')
self.update_dir_status(ds_id, result)
return result
ds_path_full_name = CFile.join_file(ds_root_path, ds_subpath)
CLogger().debug('处理的子目录为: {0}'.format(ds_path_full_name))
try:
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule,
{
'dsdStorageID': ds_storage_id,
'dsdDirectory': ds_subpath
}
)
ds_rule_content = rule_ds.value_by_name(0, 'dsdScanRule', '')
path_obj = CDMPathInfo(self.FileType_Dir, ds_path_full_name, ds_storage_id, ds_id, parent_id, owner_obj_id,
self.get_mission_db_id(), ds_rule_content)
if not path_obj.file_existed:
path_obj.db_update_status_on_path_invalid()
return CResult.merge_result(
CResult.Success,
'目录[{0}]不存在, 在设定状态后, 顺利结束!'.format(ds_path_full_name)
)
else:
path_obj.db_check_and_update_metadata_rule(
CFile.join_file(ds_path_full_name, self.FileName_MetaData_Rule)
)
path_obj.db_path2object()
result = CResult.merge_result(CResult.Success, '目录[{0}]处理顺利完成!'.format(ds_path_full_name))
self.update_dir_status(ds_id, result)
return result
except Exception as error:
result = CResult.merge_result(
CResult.Failure,
'目录[{0}]识别过程出现错误, 详细情况: {1}!'.format(ds_path_full_name, error.__str__())
)
self.update_dir_status(ds_id, result)
return result
def notify_object(self, inbound_id: str, access: str, memo: str, obj_id,
obj_name, obj_type, quality) -> str:
"""
处理数管中识别的对象, 与第三方模块的通知
. 如果第三方模块自行处理, 则无需继承本方法
. 如果第三方模块可以处理, 则在本模块中, 从数据库中提取对象的信息, 写入第三方模块的数据表中, 或者调用第三方模块接口
注意: 在本方法中, 不要用_quality_info属性, 因为外部调用方考虑的效率因素, 没有传入!!!
@:param access 当前模块对当前对象的权限
:return:
"""
ds_na = CFactory().give_me_db(self._db_id).one_row(
'''
select dsonid, dson_notify_status
from dm2_storage_obj_na
where dson_app_id = :app_id
and dson_object_id = :object_id
''', {
'app_id':
CUtils.dict_value_by_name(self.information(), self.Name_ID,
''),
'object_id':
obj_id
})
target_notify_status = self.ProcStatus_InQueue
if not CUtils.equal_ignore_case(access, self.DataAccess_Pass):
target_notify_status = self.ProcStatus_Finished
if not ds_na.is_empty():
na_id = CUtils.any_2_str(ds_na.value_by_name(0, 'dsonid', 0))
if (ds_na.value_by_name(0, 'dson_notify_status', self.ProcStatus_Finished) == self.ProcStatus_Finished) or \
(ds_na.value_by_name(0, 'dson_notify_status', self.ProcStatus_Finished) == self.ProcStatus_InQueue):
CFactory().give_me_db(self._db_id).execute(
'''
update dm2_storage_obj_na
set dson_notify_status = :status
, dson_object_access = :object_access
, dson_access_memo = :object_access_memo
, dson_inbound_id = :inbound_id
where dsonid = :id
''', {
'id': na_id,
'status': target_notify_status,
'object_access': access,
'inbound_id': inbound_id,
'object_access_memo': memo
})
else:
CFactory().give_me_db(self._db_id).execute(
'''
insert into dm2_storage_obj_na(dson_app_id, dson_object_access, dson_object_id, dson_inbound_id, dson_access_memo)
values(:app_id, :object_access, :object_id, :inbound_id, :object_access_memo)
''', {
'app_id':
CUtils.dict_value_by_name(self.information(), self.Name_ID,
''),
'object_id':
obj_id,
'object_access':
access,
'inbound_id':
inbound_id,
'object_access_memo':
memo
})
return CResult.merge_result(
self.Success, '对象[{0}]已经推送给模块[{1}]队列! '.format(
obj_name,
CUtils.dict_value_by_name(self.information(), self.Name_Title,
'')))
def plugins_classified(cls, file_info: CDMFilePathInfoEx) -> CPlugins:
"""
插件识别
1. 首先检查file_info的__rule_content__中, 有无优先识别插件列表, 有则使用该列表按顺序进行识别
1. 其次, 检查file_info的__rule_content__是否为空, 如果不为空, 则获取其类型, 如果类型不为空, 则按类型, 匹配系统配置
1. 其次, 检查应用系统配置中, 有无对目录识别插件的特殊设置, 有则按设置中的列表进行识别
1. 最后按系统插件目录下的顺序, 对数据进行识别
:param file_info:
:return:
"""
class_classified_obj = None
# 根据rule_content, 获取入库规则文件的特定插件列表
if file_info.file_type == cls.FileType_Dir:
plugin_node_list = CXml.xml_xpath(file_info.rule_content,
cls.Path_MD_Rule_Plugins_Dir)
else:
plugin_node_list = CXml.xml_xpath(file_info.rule_content,
cls.Path_MD_Rule_Plugins_File)
if len(plugin_node_list) > 0:
# 根据指定的插件列表, 尝试进行识别
class_classified_obj = cls.__plugins_classified_by_plugin_node_list__(
file_info, plugin_node_list)
if class_classified_obj is not None:
return class_classified_obj
# 如果入库规则文件中没有指定插件列表, 则读取数据入库规则类型, 它==数据集类型
rule_type = CXml.get_element_text(
CXml.xml_xpath_one(file_info.rule_content, cls.Path_MD_Rule_Type))
if not CUtils.equal_ignore_case(rule_type, ''):
# 如果有数据入库规则文件, 则获取其规则类型, 匹配setting中的类型
plugins_json_array = settings.application.xpath_one(
cls.Path_Setting_MetaData_Plugins_Dir, None)
if plugins_json_array is not None:
for plugins_define in plugins_json_array:
key_word = CUtils.any_2_str(
CUtils.dict_value_by_name(plugins_define,
cls.Name_Keyword, None))
plugin_list = CUtils.dict_value_by_name(
plugins_define, cls.Name_Plugin, None)
if plugin_list is None:
continue
if CUtils.equal_ignore_case(key_word, rule_type):
class_classified_obj = cls.__plugins_classified_by_plugin_list__(
file_info, plugin_list)
else:
# 如果没有数据入库规则文件, 则通过目录, 逐一匹配setting中的类型
file_path = file_info.file_path_with_rel_path
plugins_json_array = settings.application.xpath_one(
cls.Path_Setting_MetaData_Plugins_Dir, None)
if plugins_json_array is not None:
for plugins_define in plugins_json_array:
key_word = CUtils.any_2_str(
CUtils.dict_value_by_name(plugins_define,
cls.Name_Keyword, None))
plugin_list = CUtils.dict_value_by_name(
plugins_define, cls.Name_Plugin, None)
if plugin_list is None:
continue
# todo(注意) 如果关键字为空, 则表明所有子目录都优先使用设置的插件列表进行识别!!!
if CUtils.equal_ignore_case(key_word, ''):
class_classified_obj = cls.__plugins_classified_by_plugin_list__(
file_info, plugin_list)
else:
if CFile.subpath_in_path(CUtils.any_2_str(key_word),
file_path):
class_classified_obj = cls.__plugins_classified_by_plugin_list__(
file_info, plugin_list)
if class_classified_obj is not None:
return class_classified_obj
else:
continue
if class_classified_obj is not None:
return class_classified_obj
else:
return cls.__plugins_classified_of_directory__(file_info)
def image2view(self, target_ds, view_path: str) -> str:
"""
影像文件转jpg或png
对波段数据类型为Byte（8位无符号整型）的影像，可采用gdal.Translate()方法直接进行文件格式转换。
对波段数据类型为UInt16（16位无符号整型）、Int16（16位有符号整型）的影像：
由于JPEG不支持16位影像的转换，且PNG转换效果非常不理想，图像轮廓模糊。
因此对16位影像采用百分比截断的方法压缩至0~255的范围，改变了像素深度，降低到8位。
该方法生成的快视图的转换效果与gdal.Translate()方法生成的相比，图像轮廓清晰，可辨识度高。
注：gdal == 3.1.3
:param target_ds: 影像临时文件的数据集
:param view_path: 快视图或拇指图文件地址
:return:
"""
cols = target_ds.RasterXSize
rows = target_ds.RasterYSize
band_count = target_ds.RasterCount
band = target_ds.GetRasterBand(1)
data_type_name = gdal.GetDataTypeName(band.DataType)
# 检查影像位深度（波段数据类型）
if CUtils.equal_ignore_case(data_type_name, 'Byte'):
# 对波段数据类型为Byte（8位无符号整型）的影像进行转换
if CFile.check_and_create_directory(view_path):
if CFile.file_ext(view_path).lower() == 'jpg':
out = gdal.Translate(view_path, target_ds, format='JPEG')
elif CFile.file_ext(view_path).lower() == 'png':
out = gdal.Translate(view_path, target_ds, format='PNG')
else:
# 对波段数据类型为UInt16（16位无符号整型）、Int16（16位有符号整型）的影像进行转换
# 检查影像波段数并读取
if band_count >= 3:
bandsOrder = [3, 2, 1]
data = np.empty([rows, cols, 3], dtype=float)
for i in range(3):
band = target_ds.GetRasterBand(bandsOrder[i])
data1 = band.ReadAsArray(0, 0, cols, rows)
data[:, :, i] = data1
if CFile.check_and_create_directory(view_path):
# 百分比截断压缩影像，将像元取值限定在0~255
lower_percent = 0.6
higher_percent = 99.4
n = data.shape[2]
out = np.zeros_like(data, dtype=np.uint8)
for i in range(n):
a = 0
b = 255
c = np.percentile(data[:, :, i], lower_percent)
d = np.percentile(data[:, :, i], higher_percent)
t = a + (data[:, :, i] - c) * (b - a) / (d - c)
t[t < a] = a
t[t > b] = b
out[:, :, i] = t
outImg = Image.fromarray(np.uint8(out))
outImg.save(view_path)
else:
data = np.empty([rows, cols], dtype=float)
band = target_ds.GetRasterBand(1)
data1 = band.ReadAsArray(0, 0, cols, rows)
data[:, :] = data1
if CFile.check_and_create_directory(view_path):
# 百分比截断压缩影像，将像元取值限定在0~255
lower_percent = 0.6
higher_percent = 99.4
out = np.zeros_like(data, dtype=np.uint8)
a = 0
b = 255
c = np.percentile(data[:, :], lower_percent)
d = np.percentile(data[:, :], higher_percent)
t = a + (data[:, :] - c) * (b - a) / (d - c)
t[t < a] = a
t[t > b] = b
out[:, :] = t
outImg = Image.fromarray(np.uint8(out))
outImg.save(view_path)
del target_ds
del out
return view_path
def init_qa_metadata_bus_xml_list(self, parser: CMetaDataParser) -> list:
"""
初始化默认的, 业务元数据xml文件的检验列表
完成 负责人 王学谦
:param parser:
:return:
"""
qa_metadata_bus_xml_list = [{
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//MetaDataFileName",
self.Name_ID: 'MetaDataFileName',
self.Name_Title: '带扩展名元数据文件名',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_NotNull: True,
self.Name_DataType: self.value_type_string,
self.Name_Width: 60
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/ProductName",
self.Name_ID: 'ProductName',
self.Name_Title: '对象名称',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_NotNull: True,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/Owner",
self.Name_ID: 'Owner',
self.Name_Title: '所有者',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/Producer",
self.Name_ID: 'Producer',
self.Name_Title: '生产商',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/Publisher",
self.Name_ID: 'Publisher',
self.Name_Title: '出版商',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/ProduceDate",
self.Name_ID: 'ProduceDate',
self.Name_Title: '生产日期',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_date,
self.Name_NotNull: True
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath:
"/Metadatafile/BasicDataContent/ConfidentialLevel",
self.Name_ID: 'ConfidentialLevel',
self.Name_Title: '密级',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/GroundResolution",
self.Name_ID: 'GroundResolution',
self.Name_Title: '地面分辨率',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_NotNull: True,
self.Name_DataType: self.value_type_string,
self.Name_Width: 10
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/ImgColorModel",
self.Name_ID: 'ImgColorModel',
self.Name_Title: '影像类型',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/PixelBits",
self.Name_ID: 'PixelBits',
self.Name_Title: '位深',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 50
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "/Metadatafile/BasicDataContent/DataFormat",
self.Name_ID: 'DataFormat',
self.Name_Title: '格式',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_NotNull: True,
self.Name_DataType: self.value_type_string,
self.Name_Width: 25
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath:
"/Metadatafile/BasicDataContent/Mathfoundation/MapProjection",
self.Name_ID: 'MapProjection',
self.Name_Title: '地图投影',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 50
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//SateName",
self.Name_ID: 'SateName',
self.Name_Title: '星源',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 38
}]
try:
imgtype = parser.metadata.metadata_bus_xml(
).get_element_text_by_xpath_one('//ImgSource')
except Exception:
imgtype = None
if CUtils.equal_ignore_case(imgtype, '0'):
qa_metadata_bus_xml_list.extend([{
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//CameraType",
self.Name_ID: 'CameraType',
self.Name_Title: '航摄仪型号',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//DigitalPhotoResolution",
self.Name_ID: 'DigitalPhotoResolution',
self.Name_Title: '航片分辨率',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_decimal_or_integer,
self.Name_Width: 8
}, {
self.Name_Type:
self.QA_Type_XML_Node_Exist,
self.Name_XPath:
"//PhotoDate",
self.Name_ID:
'PhotoDate',
self.Name_Title:
'航片日期',
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Result:
self.QA_Result_Error,
self.Name_DataType:
self.value_type_date
}])
elif CUtils.equal_ignore_case(imgtype, '1'):
qa_metadata_bus_xml_list.extend([{
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//PBandSensorType",
self.Name_ID: 'PBandSensorType',
self.Name_Title: '全色传感器',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//SateResolution",
self.Name_ID: 'SateResolution',
self.Name_Title: '全色分辨率',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_decimal_or_integer,
self.Name_Width: 8
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//PbandOrbitCode",
self.Name_ID: 'PBandOribitCode',
self.Name_Title: '全色轨道号',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 38
}, {
self.Name_Type:
self.QA_Type_XML_Node_Exist,
self.Name_XPath:
"//PbandDate",
self.Name_ID:
'PbandDate',
self.Name_Title:
'全色拍摄日期',
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Result:
self.QA_Result_Error,
self.Name_DataType:
self.value_type_date
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//MultiBandSensorType",
self.Name_ID: 'MultiBandSensorType',
self.Name_Title: '多光谱传感器或航摄仪型号',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 100
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//MultiBandResolution",
self.Name_ID: 'MultiBandResolution',
self.Name_Title: '多光谱分辨率',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_decimal_or_integer,
self.Name_Width: 8
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//MultiBandOrbitCode",
self.Name_ID: 'MultiBandOrbitCode',
self.Name_Title: '多光谱轨道号',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 100
}, {
self.Name_Type:
self.QA_Type_XML_Node_Exist,
self.Name_XPath:
"//MultiBandDate",
self.Name_ID:
'MultiBandDate',
self.Name_Title:
'多光谱拍摄日期',
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Result:
self.QA_Result_Error,
self.Name_DataType:
self.value_type_decimal_or_integer
}, {
self.Name_Type:
self.QA_Type_XML_Node_Exist,
self.Name_XPath:
"//MultiBandNum",
self.Name_ID:
'MultiBandNum',
self.Name_Title:
'多光谱波段数量',
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Result:
self.QA_Result_Error,
self.Name_DataType:
self.value_type_decimal_or_integer
}, {
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//MultiBandName",
self.Name_ID: 'MultiBandName',
self.Name_Title: '多光谱波段名称',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_DataType: self.value_type_string,
self.Name_Width: 20
}])
else:
qa_metadata_bus_xml_list.extend([{
self.Name_Type: self.QA_Type_XML_Node_Exist,
self.Name_XPath: "//ImgSource",
self.Name_ID: 'ImgSource',
self.Name_Title: '影像数据源类型',
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Result: self.QA_Result_Error,
self.Name_NotNull: True,
self.Name_List: ['0', '1']
}])
return qa_metadata_bus_xml_list
def _do_access(self) -> str:
try:
quality_info_xml = self._quality_info # 获取质检xml
quality_summary = self._dataset.value_by_name(0, 'dso_quality_summary', '')
quality_summary_json = CJson()
quality_summary_json.load_obj(quality_summary)
access_wait_flag = self.DB_False # 定义等待标志，为True则存在检查项目为等待
access_forbid_flag = self.DB_False # 定义禁止标志，为True则存在检查项目为禁止
message = ''
# 文件与影像质检部分
file_qa = quality_summary_json.xpath_one('total', '')
image_qa = quality_summary_json.xpath_one('metadata.data', '')
if CUtils.equal_ignore_case(file_qa, self.QA_Result_Error) \
or CUtils.equal_ignore_case(image_qa, self.QA_Result_Error):
message = message + '[数据与其相关文件的质检存在error!请进行修正！]'
access_forbid_flag = self.DB_True
elif CUtils.equal_ignore_case(file_qa, self.QA_Result_Warn) \
or CUtils.equal_ignore_case(image_qa, self.QA_Result_Warn):
message = message + '[数据与其相关文件的质检存在warn!请进行检查！]'
access_wait_flag = self.DB_True
elif CUtils.equal_ignore_case(quality_summary, ''):
message = message + '[数据质检未进行，可能数据存在问题!请进行检查！]'
access_forbid_flag = self.DB_True
else:
pass
for qa_name, qa_id in self.access_check_dict().items(): # 循环写好的检查列表
# qa_id = CUtils.dict_value_by_name(access_check_dict, 'qa_id', '') # 获取id
qa_node = quality_info_xml.xpath_one("//item[@id='{0}']".format(qa_id)) # 查询xml中的节点
if qa_node is not None:
node_result = CXml.get_attr(qa_node, self.Name_Result, '', False) # 获取质检结果
if CUtils.equal_ignore_case(node_result, self.QA_Result_Pass):
pass
elif CUtils.equal_ignore_case(node_result, self.QA_Result_Warn): # 警告则等待
message = message + '[业务元数据的质检中，项目{0}不符合要求，建议修正！]'.format(qa_name)
access_wait_flag = self.DB_True
else: # 错误以及其他情况，比如''，或者为其他字段
message = message + '[业务元数据的质检中，项目{0}不符合要求，必须修改后方可入库！]'.format(qa_name)
access_forbid_flag = self.DB_True
else:
message = message + '[业务元数据的质检中，没有项目{0}，请进行修正！]'.format(qa_name)
access_forbid_flag = self.DB_True
# 数据库部分
access_wait_flag, access_forbid_flag, message = \
self.db_access_check(access_wait_flag, access_forbid_flag, message)
# 开始进行检查的结果判断
access_flag = self.DataAccess_Pass
if access_forbid_flag:
access_flag = self.DataAccess_Forbid
elif access_wait_flag:
access_flag = self.DataAccess_Wait
if CUtils.equal_ignore_case(message, ''):
message = '模块可以进行访问!'
result = CResult.merge_result(
self.Success,
'模块[{0}.{1}]对对象[{2}]的访问能力已经分析完毕!分析结果为:{3}'.format(
CUtils.dict_value_by_name(self.information(), self.Name_ID, ''),
CUtils.dict_value_by_name(self.information(), self.Name_Title, ''),
self._obj_name,
message
)
)
return CResult.merge_result_info(result, self.Name_Access, access_flag)
except Exception as error:
result = CResult.merge_result(
self.Failure,
'模块[{0}.{1}]对对象[{2}]的访问能力的分析存在异常!详细情况: {3}!'.format(
CUtils.dict_value_by_name(self.information(), self.Name_ID, ''),
CUtils.dict_value_by_name(self.information(), self.Name_Title, ''),
self._obj_name,
error.__str__()
)
)
return CResult.merge_result_info(result, self.Name_Access, self.DataAccess_Forbid)
def classified(self):
"""
关键字识别
"""
super().classified()
# 预获取需要的参数
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
# 预定义逻辑参数 数据文件匹配
object_file_name_flag = False
object_file_path_flag = False
object_file_ext_flag = False
object_affiliated_file_main_flag = False
object_file_affiliated_flag = False
object_keyword_list = self.get_classified_character_of_object_keyword()
if len(object_keyword_list) > 0:
for keyword_info in object_keyword_list:
keyword_id = CUtils.dict_value_by_name(keyword_info,
self.Name_ID, None)
regex_match = CUtils.dict_value_by_name(
keyword_info, self.Name_RegularExpression, '.*')
if regex_match is None:
regex_match = '.*'
if CUtils.equal_ignore_case(keyword_id, self.Name_FileName):
if CUtils.text_match_re(file_main_name, regex_match):
object_file_name_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath):
if CUtils.text_match_re(file_path, regex_match):
object_file_path_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt):
if CUtils.text_match_re(file_ext, regex_match):
object_file_ext_flag = True
else:
same_name_file_list = CFile.file_or_dir_fullname_of_path(
file_path, False,
'(?i)^' + file_main_name + '[.].*$',
CFile.MatchType_Regex)
if len(same_name_file_list) > 0:
for same_name_file in same_name_file_list:
same_name_file_ext = CFile.file_ext(
same_name_file)
if CUtils.text_match_re(
same_name_file_ext, regex_match):
object_affiliated_file_main_flag = True
elif CUtils.equal_ignore_case(keyword_id,
self.Name_FileAffiliated):
affiliated_file_path = CUtils.dict_value_by_name(
keyword_info, self.Name_FilePath, None)
if affiliated_file_path is not None:
if CFile.find_file_or_subpath_of_path(
affiliated_file_path, regex_match,
CFile.MatchType_Regex):
object_file_affiliated_flag = True
else:
object_file_affiliated_flag = True
# 预定义逻辑参数 附属文件匹配
affiliated_file_name_flag = False
affiliated_file_path_flag = False
affiliated_file_ext_flag = False
affiliated_file_main_flag = False
affiliated_keyword_list = self.get_classified_character_of_affiliated_keyword(
)
if len(affiliated_keyword_list) > 0:
for keyword_info in affiliated_keyword_list:
keyword_id = CUtils.dict_value_by_name(keyword_info,
self.Name_ID, None)
regex_match = CUtils.dict_value_by_name(
keyword_info, self.Name_RegularExpression, '.*')
if regex_match is None:
regex_match = '.*'
if CUtils.equal_ignore_case(keyword_id, self.Name_FileName):
if CUtils.text_match_re(file_main_name, regex_match):
affiliated_file_name_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath):
if CUtils.text_match_re(file_path, regex_match):
affiliated_file_path_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt):
if CUtils.text_match_re(file_ext, regex_match):
affiliated_file_ext_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileMain):
affiliated_file_path = CUtils.dict_value_by_name(
keyword_info, self.Name_FilePath, None)
if affiliated_file_path is not None:
if CFile.find_file_or_subpath_of_path(
affiliated_file_path, regex_match,
CFile.MatchType_Regex):
affiliated_file_main_flag = True
if object_file_name_flag and object_file_path_flag and \
object_file_ext_flag and object_file_affiliated_flag:
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
self.set_custom_affiliated_file()
elif affiliated_file_name_flag and affiliated_file_path_flag and \
affiliated_file_ext_flag and affiliated_file_main_flag:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
elif object_file_name_flag and object_file_path_flag and object_affiliated_file_main_flag:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
else:
self._object_confirm = self.Object_Confirm_IUnKnown
self._object_name = None
return self._object_confirm, self._object_name
def search(self, module_name: str, search_json_obj: CJson, other_option: dict = None) -> CDataSet:
"""
根据搜索条件json, 检索符合要求的对象, 并以数据集的方式返回如下字段:
1. object_id
1. object_name
1. object_type
1. object_data_type
1. object_parent_id
1. object_size
1. object_lastmodifytime
:param module_name: 模块名称
:param search_json_obj:
:param other_option:
:return:
"""
if search_json_obj is None:
return CDataSet()
params_search = dict()
sql_from = ''
sql_where = ''
if (not CUtils.equal_ignore_case(module_name, self.ModuleName_MetaData)) and \
(not CUtils.equal_ignore_case(module_name, '')):
# sql_where = "dm2_storage_object.dso_da_result#>>'{{{0},result}}'='pass'".format(module_name)
sql_from = ', dm2_storage_obj_na '
sql_where = " dm2_storage_obj_na.dson_app_id = 'module_name' "
condition_obj_access = search_json_obj.xpath_one(self.Name_Access, self.DataAccess_Pass)
if not CUtils.equal_ignore_case(condition_obj_access, ''):
condition = "dm2_storage_obj_na.dson_object_access = '{0}'".format(CUtils.any_2_str(condition_obj_access))
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_inbound_id = search_json_obj.xpath_one(self.Name_InBound, None)
if not CUtils.equal_ignore_case(condition_inbound_id, ''):
condition = "dm2_storage_obj.dso_ib_id = '{0}'".format(CUtils.any_2_str(condition_inbound_id))
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_tag = search_json_obj.xpath_one(self.Name_Tag, None)
if condition_tag is not None:
if isinstance(condition_tag, list):
condition = CUtils.list_2_str(condition_tag, "'", ", ", "'", True)
else:
condition = CUtils.list_2_str([condition_tag], "'", ", ", "'", True)
if not CUtils.equal_ignore_case(condition, ''):
condition = 'dm2_storage_object.dsotags @ > array[{0}]:: CHARACTER VARYING[]'.format(condition)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_id = search_json_obj.xpath_one(self.Name_ID, None)
if condition_id is not None:
if isinstance(condition_id, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodid', condition_id, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodid', condition_id, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_name = search_json_obj.xpath_one(self.Name_Name, None)
if condition_name is not None:
if isinstance(condition_name, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodname', condition_name, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodname', condition_name, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_type = search_json_obj.xpath_one(self.Name_Type, None)
if condition_type is not None:
if isinstance(condition_type, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodtype', condition_type, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodtype', condition_type, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_group = search_json_obj.xpath_one(self.Name_Group, None)
if condition_group is not None:
if isinstance(condition_group, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
if not CUtils.equal_ignore_case(sql_where, ''):
sql_where = ' and {0}'.format(sql_where)
sql_search = '''
select dm2_storage_object.dsoid as object_id
, dm2_storage_object.dsoobjectname as object_name
, dm2_storage_object.dsoobjecttype as object_type
, dm2_storage_object.dsodatatype as object_data_type
, dm2_storage_object.dsoparentobjid as object_parent_id
, dm2_storage_object.dso_volumn_now as object_size
, dm2_storage_object.dso_obj_lastmodifytime as object_lastmodifytime
from dm2_storage_object, dm2_storage_object_def {0}
where dm2_storage_object.dsoobjecttype = dm2_storage_object_def.dsodid
and dm2_storage_object.dsoid = dm2_storage_obj_na.dson_object_id
{1}
'''.format(sql_from, sql_where)
return CFactory().give_me_db(self.db_server_id).all_row(sql_search)
def __condition_value_like_2_sql(self, field, value):
if CUtils.equal_ignore_case(value, ''):
return ""
return "{0} like '{0}'".format(field, value)
def parser_metadata_custom(self, parser: CMetaDataParser) -> str:
"""
自定义的元数据解析, 在所有质检和其他处理之后触发
:param parser:
:return:
"""
meta_data_json = parser.metadata.metadata_json()
if meta_data_json is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 无法获取JSON格式的元数据! '.format(
self.file_info.file_name_with_full_path,
)
)
json_data_source = meta_data_json.xpath_one('datasource', None)
layer_list = meta_data_json.xpath_one(self.Name_Layers, None)
if layer_list is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 元数据中无法找到layers节点! '.format(
self.file_info.file_name_with_full_path,
)
)
mdb_ib_id = CFactory().give_me_db(self.file_info.db_server_id).one_value(
'''
select dso_ib_id
from dm2_storage_object
where dsoid = :object_id
''',
{
'object_id': parser.object_id
}
)
error_message_list = []
table = CTable()
table.load_info(self.file_info.db_server_id, self.TableName_DM_Object)
for layer in layer_list:
layer_name = CUtils.dict_value_by_name(layer, self.Name_Name, '')
if CUtils.equal_ignore_case(layer_name, ''):
continue
layer_alias_name = CUtils.dict_value_by_name(layer, self.Name_Description, layer_name)
layer_metadata_json = CJson()
layer_metadata_json.set_value_of_name('datasource', json_data_source)
layer_metadata_json.set_value_of_name('layer_count', 1)
layer_metadata_json.set_value_of_name('layers', [layer])
layer_metadata_text = layer_metadata_json.to_json()
try:
sql_find_layer_existed = '''
select dsoid as layer_id_existed
from dm2_storage_object
where upper(dsoobjectname) = upper(:layer_name)
and dsoparentobjid = :object_id
'''
layer_id_existed = CFactory().give_me_db(self.file_info.db_server_id).one_value(
sql_find_layer_existed,
{
'layer_name': layer_name,
'object_id': parser.object_id
}
)
if layer_id_existed is None:
layer_id_existed = CUtils.one_id()
table.column_list.reset()
table.column_list.column_by_name('dsoid').set_value(layer_id_existed)
table.column_list.column_by_name('dsoobjectname').set_value(layer_name)
table.column_list.column_by_name('dsoobjecttype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Plugins_Name,
''
)
)
table.column_list.column_by_name('dsodatatype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Data_Type,
''
)
)
table.column_list.column_by_name('dsoalphacode').set_value(CUtils.alpha_text(layer_name))
table.column_list.column_by_name('dsoaliasname').set_value(layer_alias_name)
table.column_list.column_by_name('dsoparentobjid').set_value(parser.object_id)
table.column_list.column_by_name('dso_ib_id').set_value(mdb_ib_id)
table.column_list.column_by_name('dsometadatatext').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadatajson').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadataparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsotagsparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsodetailparsestatus').set_value(self.ProcStatus_InQueue)
result = table.save_data()
if not CResult.result_success(result):
error_message_list.append(
'图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(
layer_name,
CResult.result_message(result)
)
)
except Exception as error:
error_message_list.append('图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(layer_name, error.__str__()))
if len(error_message_list) > 0:
return CResult.merge_result(
self.Failure,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 详细情况如下: \n{1}'.format(
self.file_info.file_name_with_full_path,
CUtils.list_2_str(error_message_list, '', '\n', '', True)
)
)
else:
return CResult.merge_result(
self.Success,
'数据[{0}]的自定义元数据解析完毕! '.format(
self.file_info.file_name_with_full_path,
)
)