def init_before_test(self):
plugins_info = self.create_plugins().get_information()
plugins_catalog = CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Catalog_Title, '')
self._test_file_root_path = settings.application.xpath_one(
self.Path_Setting_Dir_Test_Data, '')
self._test_file_parent_path = CFile.join_file(
settings.application.xpath_one(self.Path_Setting_Dir_Test_Data,
''), plugins_catalog)
def metadata_bus_dict_process_custom(self, metadata_bus_dict):
"""
对部分需要进行运算的数据进行处理
"""
super().metadata_bus_dict_process_custom(metadata_bus_dict)
productattribute = CUtils.dict_value_by_name(metadata_bus_dict,
'productattribute', None)
if not CUtils.equal_ignore_case(productattribute, ''):
if len(productattribute) > 2:
metadata_bus_dict['productattribute'] = productattribute[:2]
def run(self):
if not CUtils.equal_ignore_case(CSys.get_platform_system_name(),
self.OS_Windows):
base_signal.signal(base_signal.SIGCHLD, self.wait_child)
CLogger().info('控制中心进程[{0}]启动运行...'.format(self.pid))
# 此时, 才在controlCenter的进程中
host_settings_dict = CUtils.dict_value_by_name(
self.__shared_control_center_info__, self.NAME_CMD_SETTINGS, None)
settings.application.load_obj(host_settings_dict)
settings.application.init_sys_path()
CLogger().info('控制中心进程[{0}]启动哨兵值守进程...'.format(self.pid))
self.__control_center_manager__ = Manager()
self.__control_center_params = self.__control_center_manager__.dict()
self.__control_center_objects__ = self.__control_center_manager__.dict(
)
self.__control_center_objects_locker__ = self.__control_center_manager__.Lock(
)
self.__sentinel_manager__ = Manager()
self.__sentinel_queue__ = self.__sentinel_manager__.Queue()
self.__sentinel_stop_event__ = self.__sentinel_manager__.Event()
self.__control_center_params[
self.NAME_CMD_SETTINGS] = host_settings_dict
self.__sentinel_process__ = CSentinel(
self.__control_center_objects_locker__,
self.__control_center_objects__, self.__sentinel_stop_event__,
self.__sentinel_queue__, self.__control_center_params)
self.__sentinel_process__.daemon = True
self.__sentinel_process__.start()
while True:
CLogger().info('控制中心进程[{0}]开始检查接收任务队列...'.format(self.pid))
if self.process_queue_command(
) == self.const_command_queue_process_result_notify_terminal:
break
CLogger().info('控制中心进程[{0}]开始检查哨兵反馈的消息...'.format(self.pid))
self.process_queue_sentinel()
CLogger().info('控制中心进程[{0}]开始同步控制中心对外公布的数据...'.format(self.pid))
# 同步一下控制中心的对外公布数据
self.sync_shared_control_center_info()
# 休息一下
time.sleep(3)
CLogger().info('控制中心进程[{0}]开始进行退出前的准备工作, 该过程会比较复杂漫长...'.format(
self.pid))
# 开始处理退出工作
self.before_stop()
def batch_qa_metadata_bus_json_item(self, list_qa: list):
"""
批量处理json格式的业务元数据中的质检项目
:param list_qa:
:return:
"""
if len(list_qa) == 0:
return
for qa_item in list_qa:
list_result = CAudit.a_json_element(
CUtils.dict_value_by_name(qa_item, self.Name_ID, ''),
CUtils.dict_value_by_name(qa_item, self.Name_Title, ''),
CUtils.dict_value_by_name(qa_item, self.Name_Group, self.QA_Group_Data_Integrity),
CUtils.dict_value_by_name(qa_item, self.Name_Result, self.QA_Result_Pass),
self.metadata.metadata_bus_json(),
CUtils.dict_value_by_name(qa_item, self.Name_XPath, ''),
qa_item
)
item_result = self.get_qa_result_from_list(list_result)
self.metadata.quality.append_metadata_bus_quality(item_result)
def run(self):
# 此时, 才在worker的进程中
host_settings_dict = CUtils.dict_value_by_name(self.__params__, self.NAME_CMD_SETTINGS, None)
settings.application.load_obj(host_settings_dict)
settings.application.init_sys_path()
# 在终止进程时，部分进程将得到信号，进入运行机制，但此之前，停止信号应该已经设置！！！进程将直接结束
if not self.accept_stop_message():
self.__runner__ = self.get_or_create_worker_execute(self.__cmd_id__, self.__cmd_trigger__,
self.__cmd_algorithm__, self.__cmd_params__,
self.__stop_event__)
if self.__runner__ is not None:
self.__runner__.start()
def __plugins_classified_of_directory__(
cls, file_info: CDMFilePathInfoEx) -> CPlugins:
"""
使用系统目录下的所有插件进行识别
:param file_info:
:return:
"""
target = file_info.file_main_name
target_type = file_info.file_type
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), target_type)
path = CFile.join_file(CSys.get_plugins_root_dir(), target_type)
plugins_file_list = CFile.file_or_subpath_of_path(
path, '{0}_*.{1}'.format(cls.Name_Plugins, cls.FileExt_Py))
for file_name_without_path in plugins_file_list:
file_main_name = CFile.file_main_name(file_name_without_path)
try:
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, file_main_name, file_info)
obj_info = class_classified_obj.get_information()
# 王西亚 添加逻辑于2021-02-12, 解决项目插件调试不方便的问题
obj_classified_valid = False
obj_owner_project_id = CUtils.dict_value_by_name(
obj_info, CPlugins.Plugins_Info_Project_ID, None)
system_project_id = settings.application.xpath_one(
cls.Path_Setting_Project_ID, None)
if obj_owner_project_id is None:
obj_classified_valid = True
else:
obj_classified_valid = CUtils.equal_ignore_case(
obj_owner_project_id, system_project_id)
if obj_classified_valid:
object_confirm, object_name = class_classified_obj.classified(
)
if object_confirm != cls.Object_Confirm_IUnKnown:
obj_id = class_classified_obj.get_id()
CLogger().debug('{0} is classified as {1}.{2}'.format(
target, obj_info, obj_id))
return class_classified_obj
except Exception as error:
CLogger().debug('插件[{0}]解析出现异常, 错误信息为: [{1}], 请检查!'.format(
file_main_name, error.__str__()))
if settings.application.xpath_one(
'{0}.{1}'.format(cls.Name_Application, cls.Name_Debug),
cls.DB_False) == cls.DB_True:
raise
else:
continue
else:
return None
def __find_column_data_set_method(self, column_obj: CColumn, data_source: CDataSetSeqReader):
pipe_list = CUtils.dict_value_by_name(self.__config, self.Name_Pipe, None)
option = CUtils.dict_value_by_name(self.__config, self.Name_Option, None)
same_field_name_match = CUtils.dict_value_by_name(option, 'same_field_name_match', True)
if pipe_list is not None:
for pipe in pipe_list:
if CUtils.equal_ignore_case(
CUtils.dict_value_by_name(pipe, self.Name_Field, None),
column_obj.name
):
return pipe
if same_field_name_match:
for field_index in range(data_source.field_count()):
field_name = data_source.field_name(field_index)
if CUtils.equal_ignore_case(field_name, column_obj.name):
return {
self.Name_Field: column_obj.name,
self.Name_Type: self.Name_Common,
self.Name_Value: '${0}'.format(field_name)
}
else:
return None
def run(self):
# 此时, 才在sentinel的进程中
host_settings_dict = CUtils.dict_value_by_name(
self.__control_center_params, self.NAME_CMD_SETTINGS, None)
settings.application.load_obj(host_settings_dict)
settings.application.init_sys_path()
while True:
# 在终止进程时，部分进程将得到信号，进入运行机制，但此之前，停止信号应该已经设置！！！进程将直接结束
if self.__stop_event__.is_set():
return
self.check_process_alive()
time.sleep(10)
def batch_qa_file(self, list_qa: list):
"""
批量处理数据完整性方面的质检项目
:param list_qa:
:return:
"""
if len(list_qa) == 0:
return
for qa_item in list_qa:
list_result = CAudit.a_file(
CUtils.dict_value_by_name(qa_item, self.Name_ID, ''),
CUtils.dict_value_by_name(qa_item, self.Name_Title, ''),
CUtils.dict_value_by_name(qa_item, self.Name_Group, self.QA_Group_Data_Integrity),
CUtils.dict_value_by_name(qa_item, self.Name_Result, self.QA_Result_Pass),
CFile.join_file(
self.file_content.content_root_dir,
CUtils.dict_value_by_name(qa_item, self.Name_FileName, '')
),
qa_item
)
item_result = self.get_qa_result_from_list(list_result)
self.metadata.quality.append_total_quality(item_result)
def begin_compress(self):
"""
开始压缩
-p后面接的是压缩到那个地方的路径,默认是压缩到当前工作的目录
-m后面接的是一个数字0或者1表示压缩方式,默认是0 1 = zipfile.ZIP_STORED , 0 = zipfile.ZIP_DEFLATED
:return:
"""
parser = argparse.ArgumentParser(
description='add zip path, eg:"/user/bin/src/"')
parser.add_argument("-p",
help='path no name, eg:"/user/bin/src/"',
default=CFile.file_abs_path('.') +
CFile.unify_seperator + "zip" +
CFile.unify_seperator)
parser.add_argument(
"-m",
help=
'zip methods, use int, eg: 1 = zipfile.ZIP_STORED and 0 = zipfile.ZIP_DEFLATED',
type=int,
default=0)
args = parser.parse_args()
path = args.p
zip_way = args.m
plugin_path = self.search_type()
print("开始创建目录.........")
print("创建目录成功")
for plugin_name in plugin_path:
path_with_name = path + CUtils.dict_value_by_name(
plugin_name, 'type',
'') + CFile.unify_seperator + CUtils.dict_value_by_name(
plugin_name, 'target', '')
print("压缩路径是%s" % path_with_name)
test = CZip_ZipFile(path_with_name)
test.new(zip_way)
test.add_file_or_path(plugin_name)
test.close()
print("所有插件压缩完成")
def metadata_bus_dict_process_custom(self, metadata_bus_dict):
"""
对部分需要进行运算的数据进行处理
"""
super().metadata_bus_dict_process_custom(metadata_bus_dict)
resolution = CUtils.dict_value_by_name(metadata_bus_dict, 'resolution',
None)
if not CUtils.equal_ignore_case(resolution, ''):
resolution_list = re.split(r'[/]|\s+', resolution.strip())
if len(resolution_list) > 0:
temp_resolution_list = []
for temp_resolution in resolution_list:
temp_resolution_list.append(
CUtils.to_decimal(temp_resolution))
metadata_bus_dict['resolution'] = min(temp_resolution_list)
def __append_quality_info(self, xml_node, audit_result: dict):
quality_id = CUtils.dict_value_by_name(audit_result, self.Name_ID, '')
quality_title = CUtils.dict_value_by_name(audit_result,
self.Name_Title, '')
quality_group = CUtils.dict_value_by_name(audit_result,
self.Name_Group,
self.QA_Group_Data_Integrity)
quality_result = CUtils.dict_value_by_name(audit_result,
self.Name_Result,
self.QA_Result_Pass)
quality_memo = CUtils.dict_value_by_name(audit_result,
self.Name_Message, '')
temp_node = CXml.node_xpath_one(
xml_node, './{0}[@id="{1}"]'.format(self.Name_Item, quality_id))
if temp_node is not None:
CXml.remove(temp_node)
temp_node = CXml.create_element(xml_node, self.Name_Item)
CXml.set_attr(temp_node, self.Name_ID, quality_id)
CXml.set_attr(temp_node, self.Name_Group, quality_group)
CXml.set_attr(temp_node, self.Name_Title, quality_title)
CXml.set_attr(temp_node, self.Name_Result, quality_result)
CXml.set_element_text(temp_node, quality_memo)
def process_mission(self, dataset) -> str:
"""
:param dataset:
:return:
"""
ds_na_id = dataset.value_by_name(0, 'na_id', '')
ds_app_id = dataset.value_by_name(0, 'app_id', '')
ds_object_id = dataset.value_by_name(0, 'object_id', '')
ds_object_type = dataset.value_by_name(0, 'object_type', '')
ds_object_name = dataset.value_by_name(0, 'object_name', '')
ds_object_access = dataset.value_by_name(0, 'object_access',
self.DataAccess_Forbid)
CLogger().debug('与第三方模块[{0}]同步的对象为: [{1}]'.format(
ds_app_id, ds_object_name))
try:
module_file_name = CFile.join_file(
CSys.get_metadata_data_access_modules_root_dir(),
'{0}.{1}'.format(ds_app_id, self.FileExt_Py))
if not CFile.file_or_path_exist(module_file_name):
message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id)
result = CResult.merge_result(self.Success, message)
self.update_sync_result(ds_na_id, result)
return result
module_obj = CObject.create_module_instance(
CSys.get_metadata_data_access_modules_root_name(), ds_app_id,
self.get_mission_db_id())
if module_obj is None:
message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id)
result = CResult.merge_result(self.Success, message)
self.update_sync_result(ds_na_id, result)
return result
module_title = CUtils.dict_value_by_name(module_obj.information(),
self.Name_Title, '')
result = module_obj.sync(ds_object_access, ds_object_id,
ds_object_name, ds_object_type, None)
self.update_sync_result(ds_na_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure,
'与第三方模块[{0}]同步的对象: [{1}]的同步过程出现异常, 详细情况: [{2}]!'.format(
ds_app_id, ds_object_name, error.__str__()))
self.update_sync_result(ds_na_id, result)
return result
def give_me_db(self, db_id=None) -> CDataBase:
if db_id is None:
rt_db_id = self.DB_Server_ID_Default
else:
rt_db_id = CUtils.any_2_str(db_id)
if CUtils.equal_ignore_case(rt_db_id, ''):
rt_db_id = self.DB_Server_ID_Default
databases = settings.application.xpath_one(self.Name_DataBases, None)
if databases is None:
raise Exception('系统未配置数据库定义, 请检查修正后重试！')
for database in databases:
if rt_db_id == CUtils.dict_value_by_name(database, self.Name_ID, self.DB_Server_ID_Default):
return self.create_db(database)
raise Exception('未找到标示为[{0}]]的数据库定义！'.format(rt_db_id))
def get_ib_schema(self, dataset_type, ib_option):
"""
根据数据集类型, 获取系统设置中的最佳入库模式
:param ib_option: 其他入库的特殊要求
:param dataset_type:
:return:
"""
ib_schema_list = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_Schema_Special, [])
ib_schema_default = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_Schema_Default, None)
for ib_schema in ib_schema_list:
ib_schema_id = CUtils.dict_value_by_name(ib_schema, self.Name_ID,
'')
if CUtils.equal_ignore_case(ib_schema_id, dataset_type):
return ib_schema
else:
return ib_schema_default
def batch_qa_metadata_bus_dict(self, metadata_bus_dict: dict, qa_sat_metadata_bus_list):
"""
批量处理json格式的业务元数据中的质检项目
:return:
"""
if len(metadata_bus_dict) == 0:
return
for qa_item in qa_sat_metadata_bus_list:
list_result = CAudit.a_dict_element(
CUtils.dict_value_by_name(qa_item, self.Name_ID, ''),
CUtils.dict_value_by_name(qa_item, self.Name_Title, ''),
CUtils.dict_value_by_name(qa_item, self.Name_Group, self.QA_Group_Data_Integrity),
CUtils.dict_value_by_name(qa_item, self.Name_Result, self.QA_Result_Pass),
CUtils.dict_value_by_name(metadata_bus_dict, CUtils.dict_value_by_name(qa_item, self.Name_ID, ''), ''),
qa_item
)
item_result = self.get_qa_result_from_list(list_result)
self.metadata.quality.append_metadata_bus_quality(item_result)
def __find_module_obj(self, obj_id):
sql_query = '''
SELECT
dm2_storage_file.dsfid as query_file_id,
dm2_storage_file.dsfdirectoryid as query_directory_id,
dm2_storage_directory.dsddirlastmodifytime as query_directory_lastmodifytime,
dm2_storage_directory.dsdid as query_dataset_directory_id,
dm2_storage_object.*
FROM
dm2_storage_object
LEFT JOIN dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid
LEFT JOIN dm2_storage_directory on dm2_storage_directory.dsd_object_id = dm2_storage_object.dsoid
WHERE
dm2_storage_object.dsoid = '{0}'
'''.format(obj_id)
db_id = self._db_id # 数据库连接标识（查询的dm2的数管库）
dataset = CFactory().give_me_db(db_id).one_row(sql_query)
object_id = dataset.value_by_name(0, 'dsoid', '')
object_name = dataset.value_by_name(0, 'dsoobjectname', '')
quality_info = dataset.value_by_name(0, 'dso_quality', '')
quality_info_xml = CXml()
quality_info_xml.load_xml(quality_info) # 加载查询出来的xml
db_id_distribution = self.DB_Server_ID_Distribution # 同步处理的目标数据库标识id
# 构建数据对象object对应的识别插件，获取get_information里面的Plugins_Info_Module_Distribute_Engine信息
class_classified_obj = CObject.get_plugins_instance_by_object_id(
db_id, object_id)
if class_classified_obj is not None:
plugins_info = class_classified_obj.get_information()
obj_type_code = CUtils.dict_value_by_name(
plugins_info, class_classified_obj.Plugins_Info_Type_Code, '')
distribution_file_main_name = CUtils.dict_value_by_name(
plugins_info,
class_classified_obj.Plugins_Info_Module_Distribute_Engine,
'') # 对应同步文件的类名称
# 判断同步插件文件是否存在
access_modules_root_dir = CSys.get_metadata_data_access_modules_root_dir(
)
distribution_root_dir = CFile.join_file(access_modules_root_dir,
self.Name_Distribution)
distribution_file = CFile.join_file(
distribution_root_dir,
'{0}.py'.format(distribution_file_main_name))
if CFile.file_or_path_exist(distribution_file):
# 构建同步对象
distribution_obj = CObject.create_module_distribution_instance(
'{0}.{1}'.format(
CSys.get_metadata_data_access_modules_root_name(),
self.Name_Distribution), distribution_file_main_name,
db_id_distribution, object_id, object_name, obj_type_code,
quality_info_xml, dataset)
distribution_obj.set_class_plugins(class_classified_obj)
result = CResult.merge_result(self.Success, '数据的信息提取完成')
return distribution_obj, result
else:
result = CResult.merge_result(self.Failure,
'同步失败，因为本类数据的同步模块未建立')
return None, result
else:
result = CResult.merge_result(self.Failure, '同步失败，因为本类数据的识别模块缺失')
return None, result
def get_metadata_bus_configuration_list(self) -> list:
"""
固定的列表，重写时不可缺项
"""
return [
{
self.Name_ID:
'satelliteid', # 卫星，必填，从元数据组织定义，必须是标准命名的卫星名称
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Scene_Source/MISSION'
},
{
self.Name_ID: 'sensorid', # 传感器 必填,从元数据组织定义，必须是标准命名的传感器名称
self.Name_Value: 'PMS'
},
{
self.Name_ID: 'centerlatitude', # 中心维度
self.Name_Value: None
},
{
self.Name_ID: 'centerlongitude', # 中心经度
self.Name_Value: None
},
{
self.Name_ID:
'topleftlatitude', # 左上角维度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[1]/FRAME_LAT'
},
{
self.Name_ID:
'topleftlongitude', # 左上角经度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[1]/FRAME_LON'
},
{
self.Name_ID:
'toprightlatitude', # 右上角维度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[2]/FRAME_LAT'
},
{
self.Name_ID:
'toprightlongitude', # 右上角经度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[2]/FRAME_LON'
},
{
self.Name_ID:
'bottomrightlatitude', # 右下角维度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[3]/FRAME_LAT'
},
{
self.Name_ID:
'bottomrightlongitude', # 右下角经度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[3]/FRAME_LON'
},
{
self.Name_ID:
'bottomleftlatitude', # 左下角维度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[4]/FRAME_LAT'
},
{
self.Name_ID:
'bottomleftlongitude', # 左下角经度 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Source_Frame/Vertex[4]/FRAME_LON'
},
{
self.Name_ID: 'transformimg', # 斜视图,可空,不用质检
self.Name_Value: None
},
{
self.Name_ID:
'centertime', # 影像获取时间 必填
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Scene_Source/START_TIME'
},
{
self.Name_ID: 'resolution', # 分辨率(米) 对应卫星的默认值，从info里取
self.Name_Value: '1/4'
},
{
self.Name_ID:
'rollangle', # 侧摆角
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Scene_Source/VIEWING_ANGLE'
},
{
self.Name_ID:
'cloudpercent', # 云量
self.Name_XPath:
'/Dimap_Document/Dataset_Sources/Source_Information/Quality_Assessment'
'/Quality_Parameter[7]/QUALITY_PARAMETER_VALUE'
},
{
self.Name_ID: 'dataum', # 坐标系 默认为null
self.Name_Value: 'WGS_1984'
},
{
self.Name_ID: 'acquisition_id', # 轨道号
self.Name_Value: None
},
{
self.Name_ID: 'copyright', # 发布来源 从info取
self.Name_XPath: '/Dimap_Document/Dataset_Id/COPYRIGHT'
},
{
self.Name_ID:
'publishdate', # 发布时间 必填
self.Name_XPath:
'/Dimap_Document/Production/DATASET_PRODUCTION_DATE'
},
{
self.Name_ID: 'remark', # 备注 可空
self.Name_Value: None
},
{
self.Name_ID: 'productname', # 产品名称，有的能从卫星元数据里面取，没有就不取
self.Name_XPath: '/Dimap_Document/Dataset_Id/DATASET_NAME'
},
{
self.Name_ID:
'producttype', # 产品类型 必填
self.Name_Value:
CUtils.dict_value_by_name(self.get_information(),
self.Plugins_Info_ProductType, None)
},
{
self.Name_ID: 'productattribute', # 产品属性 必填
self.Name_XPath: '/Dimap_Document/Production/PRODUCT_TYPE',
self.Name_Map: { # 映射，当取到的值为key时，将值转换为value
'L1B': 'L1',
'L2B': 'L2',
'L4B': 'L4'
}
},
{
self.Name_ID: 'productid', # 产品id 默认取主文件全名
self.Name_XPath: '/Dimap_Document/Production/JOB_ID'
},
{
self.Name_ID: 'otherxml', # 预留字段，可空，配置正则
self.Name_Value: None
}
]
def plugins_classified(cls, file_info: CDMFilePathInfoEx) -> CPlugins:
"""
插件识别
1. 首先检查file_info的__rule_content__中, 有无优先识别插件列表, 有则使用该列表按顺序进行识别
1. 其次, 检查file_info的__rule_content__是否为空, 如果不为空, 则获取其类型, 如果类型不为空, 则按类型, 匹配系统配置
1. 其次, 检查应用系统配置中, 有无对目录识别插件的特殊设置, 有则按设置中的列表进行识别
1. 最后按系统插件目录下的顺序, 对数据进行识别
:param file_info:
:return:
"""
class_classified_obj = None
# 根据rule_content, 获取入库规则文件的特定插件列表
if file_info.file_type == cls.FileType_Dir:
plugin_node_list = CXml.xml_xpath(file_info.rule_content,
cls.Path_MD_Rule_Plugins_Dir)
else:
plugin_node_list = CXml.xml_xpath(file_info.rule_content,
cls.Path_MD_Rule_Plugins_File)
if len(plugin_node_list) > 0:
# 根据指定的插件列表, 尝试进行识别
class_classified_obj = cls.__plugins_classified_by_plugin_node_list__(
file_info, plugin_node_list)
if class_classified_obj is not None:
return class_classified_obj
# 如果入库规则文件中没有指定插件列表, 则读取数据入库规则类型, 它==数据集类型
rule_type = CXml.get_element_text(
CXml.xml_xpath_one(file_info.rule_content, cls.Path_MD_Rule_Type))
if not CUtils.equal_ignore_case(rule_type, ''):
# 如果有数据入库规则文件, 则获取其规则类型, 匹配setting中的类型
plugins_json_array = settings.application.xpath_one(
cls.Path_Setting_MetaData_Plugins_Dir, None)
if plugins_json_array is not None:
for plugins_define in plugins_json_array:
key_word = CUtils.any_2_str(
CUtils.dict_value_by_name(plugins_define,
cls.Name_Keyword, None))
plugin_list = CUtils.dict_value_by_name(
plugins_define, cls.Name_Plugin, None)
if plugin_list is None:
continue
if CUtils.equal_ignore_case(key_word, rule_type):
class_classified_obj = cls.__plugins_classified_by_plugin_list__(
file_info, plugin_list)
else:
# 如果没有数据入库规则文件, 则通过目录, 逐一匹配setting中的类型
file_path = file_info.file_path_with_rel_path
plugins_json_array = settings.application.xpath_one(
cls.Path_Setting_MetaData_Plugins_Dir, None)
if plugins_json_array is not None:
for plugins_define in plugins_json_array:
key_word = CUtils.any_2_str(
CUtils.dict_value_by_name(plugins_define,
cls.Name_Keyword, None))
plugin_list = CUtils.dict_value_by_name(
plugins_define, cls.Name_Plugin, None)
if plugin_list is None:
continue
# todo(注意) 如果关键字为空, 则表明所有子目录都优先使用设置的插件列表进行识别!!!
if CUtils.equal_ignore_case(key_word, ''):
class_classified_obj = cls.__plugins_classified_by_plugin_list__(
file_info, plugin_list)
else:
if CFile.subpath_in_path(CUtils.any_2_str(key_word),
file_path):
class_classified_obj = cls.__plugins_classified_by_plugin_list__(
file_info, plugin_list)
if class_classified_obj is not None:
return class_classified_obj
else:
continue
if class_classified_obj is not None:
return class_classified_obj
else:
return cls.__plugins_classified_of_directory__(file_info)
def get_metadata_bus_configuration_list(self) -> list:
"""
固定的列表，重写时不可缺项
"""
return [
{
self.Name_ID: 'satelliteid', # 卫星，必填，从元数据组织定义，必须是标准命名的卫星名称
self.Name_XPath: '/Auxiliary/General/Satellite'
},
{
self.Name_ID: 'sensorid', # 传感器 必填,从元数据组织定义，必须是标准命名的传感器名称
self.Name_Value: 'PMS'
},
{
self.Name_ID: 'centerlatitude', # 中心维度
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogCenter/Latitude'
},
{
self.Name_ID: 'centerlongitude', # 中心经度
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogCenter/Longitude'
},
{
self.Name_ID: 'topleftlatitude', # 左上角维度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogTL/Latitude'
},
{
self.Name_ID: 'topleftlongitude', # 左上角经度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogTL/Longitude'
},
{
self.Name_ID: 'toprightlatitude', # 右上角维度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogTR/Latitude'
},
{
self.Name_ID: 'toprightlongitude', # 右上角经度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogTR/Longitude'
},
{
self.Name_ID: 'bottomrightlatitude', # 右下角维度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogBR/Latitude'
},
{
self.Name_ID: 'bottomrightlongitude', # 右下角经度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogBR/Longitude'
},
{
self.Name_ID: 'bottomleftlatitude', # 左下角维度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogBL/Latitude'
},
{
self.Name_ID: 'bottomleftlongitude', # 左下角经度 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingCoordinates/ImageGeogBL/Longitude'
},
{
self.Name_ID: 'transformimg', # 斜视图,可空,不用质检
self.Name_Value: None
},
{
self.Name_ID: 'centertime', # 影像获取时间 必填
self.Name_XPath: '/Auxiliary/Image/PAN/ImagingTime/ImagingCenterTime/UTC'
},
{
self.Name_ID: 'resolution', # 分辨率(米) 对应卫星的默认值，从info里取
self.Name_Value: '0.7/1'
},
{
self.Name_ID: 'rollangle', # 侧摆角
self.Name_XPath: '/Auxiliary/Image/PAN/Angle/Roll'
},
{
self.Name_ID: 'cloudpercent', # 云量
self.Name_XPath: '/Auxiliary/Image/PAN/CloudCover/Average'
},
{
self.Name_ID: 'dataum', # 坐标系 默认为null
self.Name_Value: 'WGS_1984'
},
{
self.Name_ID: 'acquisition_id', # 轨道号
self.Name_Value: None
},
{
self.Name_ID: 'copyright', # 发布来源 从info取
self.Name_Value: CUtils.dict_value_by_name(self.get_information(), self.Plugins_Info_CopyRight, None)
},
{
self.Name_ID: 'publishdate', # 发布时间 必填
self.Name_XPath: '/Auxiliary/General/CreateDate'
},
{
self.Name_ID: 'remark', # 备注 可空
self.Name_Value: None
},
{
self.Name_ID: 'productname', # 产品名称，有的能从卫星元数据里面取，没有就不取
self.Name_XPath: None
},
{
self.Name_ID: 'producttype', # 产品类型 必填
self.Name_Value: CUtils.dict_value_by_name(self.get_information(), self.Plugins_Info_ProductType, None)
},
{
self.Name_ID: 'productattribute', # 产品属性 必填
self.Name_XPath: '/Auxiliary/General/ProductLevel',
self.Name_Map: { # 映射，当取到的值为key时，将值转换为value
'Level1R': 'L1',
'Level2R': 'L2',
'Level4R': 'L4'
}
},
{
self.Name_ID: 'productid', # 产品id 默认取主文件全名
self.Name_XPath: None
},
{
self.Name_ID: 'otherxml', # 预留字段，可空，配置正则
self.Name_Value: None
}
]
def notify_object(self, inbound_id: str, access: str, memo: str, obj_id,
obj_name, obj_type, quality) -> str:
"""
处理数管中识别的对象, 与第三方模块的通知
. 如果第三方模块自行处理, 则无需继承本方法
. 如果第三方模块可以处理, 则在本模块中, 从数据库中提取对象的信息, 写入第三方模块的数据表中, 或者调用第三方模块接口
注意: 在本方法中, 不要用_quality_info属性, 因为外部调用方考虑的效率因素, 没有传入!!!
@:param access 当前模块对当前对象的权限
:return:
"""
ds_na = CFactory().give_me_db(self._db_id).one_row(
'''
select dsonid, dson_notify_status
from dm2_storage_obj_na
where dson_app_id = :app_id
and dson_object_id = :object_id
''', {
'app_id':
CUtils.dict_value_by_name(self.information(), self.Name_ID,
''),
'object_id':
obj_id
})
target_notify_status = self.ProcStatus_InQueue
if not CUtils.equal_ignore_case(access, self.DataAccess_Pass):
target_notify_status = self.ProcStatus_Finished
if not ds_na.is_empty():
na_id = CUtils.any_2_str(ds_na.value_by_name(0, 'dsonid', 0))
if (ds_na.value_by_name(0, 'dson_notify_status', self.ProcStatus_Finished) == self.ProcStatus_Finished) or \
(ds_na.value_by_name(0, 'dson_notify_status', self.ProcStatus_Finished) == self.ProcStatus_InQueue):
CFactory().give_me_db(self._db_id).execute(
'''
update dm2_storage_obj_na
set dson_notify_status = :status
, dson_object_access = :object_access
, dson_access_memo = :object_access_memo
, dson_inbound_id = :inbound_id
where dsonid = :id
''', {
'id': na_id,
'status': target_notify_status,
'object_access': access,
'inbound_id': inbound_id,
'object_access_memo': memo
})
else:
CFactory().give_me_db(self._db_id).execute(
'''
insert into dm2_storage_obj_na(dson_app_id, dson_object_access, dson_object_id, dson_inbound_id, dson_access_memo)
values(:app_id, :object_access, :object_id, :inbound_id, :object_access_memo)
''', {
'app_id':
CUtils.dict_value_by_name(self.information(), self.Name_ID,
''),
'object_id':
obj_id,
'object_access':
access,
'inbound_id':
inbound_id,
'object_access_memo':
memo
})
return CResult.merge_result(
self.Success, '对象[{0}]已经推送给模块[{1}]队列! '.format(
obj_name,
CUtils.dict_value_by_name(self.information(), self.Name_Title,
'')))
def _do_access(self) -> str:
try:
quality_info_xml = self._quality_info # 获取质检xml
quality_summary = self._dataset.value_by_name(0, 'dso_quality_summary', '')
quality_summary_json = CJson()
quality_summary_json.load_obj(quality_summary)
access_wait_flag = self.DB_False # 定义等待标志，为True则存在检查项目为等待
access_forbid_flag = self.DB_False # 定义禁止标志，为True则存在检查项目为禁止
message = ''
# 文件与影像质检部分
file_qa = quality_summary_json.xpath_one('total', '')
image_qa = quality_summary_json.xpath_one('metadata.data', '')
if CUtils.equal_ignore_case(file_qa, self.QA_Result_Error) \
or CUtils.equal_ignore_case(image_qa, self.QA_Result_Error):
message = message + '[数据与其相关文件的质检存在error!请进行修正！]'
access_forbid_flag = self.DB_True
elif CUtils.equal_ignore_case(file_qa, self.QA_Result_Warn) \
or CUtils.equal_ignore_case(image_qa, self.QA_Result_Warn):
message = message + '[数据与其相关文件的质检存在warn!请进行检查！]'
access_wait_flag = self.DB_True
elif CUtils.equal_ignore_case(quality_summary, ''):
message = message + '[数据质检未进行，可能数据存在问题!请进行检查！]'
access_forbid_flag = self.DB_True
else:
pass
for qa_name, qa_id in self.access_check_dict().items(): # 循环写好的检查列表
# qa_id = CUtils.dict_value_by_name(access_check_dict, 'qa_id', '') # 获取id
qa_node = quality_info_xml.xpath_one("//item[@id='{0}']".format(qa_id)) # 查询xml中的节点
if qa_node is not None:
node_result = CXml.get_attr(qa_node, self.Name_Result, '', False) # 获取质检结果
if CUtils.equal_ignore_case(node_result, self.QA_Result_Pass):
pass
elif CUtils.equal_ignore_case(node_result, self.QA_Result_Warn): # 警告则等待
message = message + '[业务元数据的质检中，项目{0}不符合要求，建议修正！]'.format(qa_name)
access_wait_flag = self.DB_True
else: # 错误以及其他情况，比如''，或者为其他字段
message = message + '[业务元数据的质检中，项目{0}不符合要求，必须修改后方可入库！]'.format(qa_name)
access_forbid_flag = self.DB_True
else:
message = message + '[业务元数据的质检中，没有项目{0}，请进行修正！]'.format(qa_name)
access_forbid_flag = self.DB_True
# 数据库部分
access_wait_flag, access_forbid_flag, message = \
self.db_access_check(access_wait_flag, access_forbid_flag, message)
# 开始进行检查的结果判断
access_flag = self.DataAccess_Pass
if access_forbid_flag:
access_flag = self.DataAccess_Forbid
elif access_wait_flag:
access_flag = self.DataAccess_Wait
if CUtils.equal_ignore_case(message, ''):
message = '模块可以进行访问!'
result = CResult.merge_result(
self.Success,
'模块[{0}.{1}]对对象[{2}]的访问能力已经分析完毕!分析结果为:{3}'.format(
CUtils.dict_value_by_name(self.information(), self.Name_ID, ''),
CUtils.dict_value_by_name(self.information(), self.Name_Title, ''),
self._obj_name,
message
)
)
return CResult.merge_result_info(result, self.Name_Access, access_flag)
except Exception as error:
result = CResult.merge_result(
self.Failure,
'模块[{0}.{1}]对对象[{2}]的访问能力的分析存在异常!详细情况: {3}!'.format(
CUtils.dict_value_by_name(self.information(), self.Name_ID, ''),
CUtils.dict_value_by_name(self.information(), self.Name_Title, ''),
self._obj_name,
error.__str__()
)
)
return CResult.merge_result_info(result, self.Name_Access, self.DataAccess_Forbid)
def classified(self):
"""
关键字识别
"""
super().classified()
# 预获取需要的参数
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
# 预定义逻辑参数 数据文件匹配
object_file_name_flag = False
object_file_path_flag = False
object_file_ext_flag = False
object_affiliated_file_main_flag = False
object_file_affiliated_flag = False
object_keyword_list = self.get_classified_character_of_object_keyword()
if len(object_keyword_list) > 0:
for keyword_info in object_keyword_list:
keyword_id = CUtils.dict_value_by_name(keyword_info,
self.Name_ID, None)
regex_match = CUtils.dict_value_by_name(
keyword_info, self.Name_RegularExpression, '.*')
if regex_match is None:
regex_match = '.*'
if CUtils.equal_ignore_case(keyword_id, self.Name_FileName):
if CUtils.text_match_re(file_main_name, regex_match):
object_file_name_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath):
if CUtils.text_match_re(file_path, regex_match):
object_file_path_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt):
if CUtils.text_match_re(file_ext, regex_match):
object_file_ext_flag = True
else:
same_name_file_list = CFile.file_or_dir_fullname_of_path(
file_path, False,
'(?i)^' + file_main_name + '[.].*$',
CFile.MatchType_Regex)
if len(same_name_file_list) > 0:
for same_name_file in same_name_file_list:
same_name_file_ext = CFile.file_ext(
same_name_file)
if CUtils.text_match_re(
same_name_file_ext, regex_match):
object_affiliated_file_main_flag = True
elif CUtils.equal_ignore_case(keyword_id,
self.Name_FileAffiliated):
affiliated_file_path = CUtils.dict_value_by_name(
keyword_info, self.Name_FilePath, None)
if affiliated_file_path is not None:
if CFile.find_file_or_subpath_of_path(
affiliated_file_path, regex_match,
CFile.MatchType_Regex):
object_file_affiliated_flag = True
else:
object_file_affiliated_flag = True
# 预定义逻辑参数 附属文件匹配
affiliated_file_name_flag = False
affiliated_file_path_flag = False
affiliated_file_ext_flag = False
affiliated_file_main_flag = False
affiliated_keyword_list = self.get_classified_character_of_affiliated_keyword(
)
if len(affiliated_keyword_list) > 0:
for keyword_info in affiliated_keyword_list:
keyword_id = CUtils.dict_value_by_name(keyword_info,
self.Name_ID, None)
regex_match = CUtils.dict_value_by_name(
keyword_info, self.Name_RegularExpression, '.*')
if regex_match is None:
regex_match = '.*'
if CUtils.equal_ignore_case(keyword_id, self.Name_FileName):
if CUtils.text_match_re(file_main_name, regex_match):
affiliated_file_name_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FilePath):
if CUtils.text_match_re(file_path, regex_match):
affiliated_file_path_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileExt):
if CUtils.text_match_re(file_ext, regex_match):
affiliated_file_ext_flag = True
elif CUtils.equal_ignore_case(keyword_id, self.Name_FileMain):
affiliated_file_path = CUtils.dict_value_by_name(
keyword_info, self.Name_FilePath, None)
if affiliated_file_path is not None:
if CFile.find_file_or_subpath_of_path(
affiliated_file_path, regex_match,
CFile.MatchType_Regex):
affiliated_file_main_flag = True
if object_file_name_flag and object_file_path_flag and \
object_file_ext_flag and object_file_affiliated_flag:
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
self.set_custom_affiliated_file()
elif affiliated_file_name_flag and affiliated_file_path_flag and \
affiliated_file_ext_flag and affiliated_file_main_flag:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
elif object_file_name_flag and object_file_path_flag and object_affiliated_file_main_flag:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
else:
self._object_confirm = self.Object_Confirm_IUnKnown
self._object_name = None
return self._object_confirm, self._object_name
def add_file_or_path(self, file: dict):
file_src = CUtils.dict_value_by_name(file, CResource.Name_Source, None)
for path in file_src:
self.__zip_obj__.write(path)
print("把文件%s归档到压缩文件%s中" % (path, self.__file_name__))
def get_metadata_bus_configuration_list(self) -> list:
"""
固定的列表，重写时不可缺项
self.Name_ID：字段的名称 例：self.Name_ID: 'satelliteid'
self.Name_XPath：需要从xml中取值时的xpath 例：self.Name_XPath: '/ProductMetaData/SatelliteID'
self.Name_Custom_Item：对于字段resolution做的个性化配置，将从配置的列表中取出最小的值
例：self.Name_Custom_Item: ['/ProductMetaData/ImageGSDLine','/ProductMetaData/ImageGSD',4]
self.Name_Value：不在xml取得默认值与当XPath取不到值时取的值 例 self.Name_Value: 1
self.Name_Map：映射，当取到的值为key的值时将值转换为value
例 self.Name_Map: { # 映射，当取到的值为key时，将值转换为value
'LEVEL1A': 'L1',
'LEVEL2A': 'L2',
'LEVEL4A': 'L4'
# self.Name_Default: None # 没有对应的的映射使用的默认值}
"""
return [
{
self.Name_ID: 'satelliteid', # 卫星，必填，从元数据组织定义，必须是标准命名的卫星名称
self.Name_XPath: '/ProductMetaData/SatelliteID'
},
{
self.Name_ID: 'sensorid', # 传感器 必填,从元数据组织定义，必须是标准命名的传感器名称
self.Name_XPath: '/ProductMetaData/SensorID'
},
{
self.Name_ID: 'centerlatitude', # 中心维度
self.Name_Value: None
},
{
self.Name_ID: 'centerlongitude', # 中心经度
self.Name_Value: None
},
{
self.Name_ID: 'topleftlatitude', # 左上角维度 必填
self.Name_XPath: '/ProductMetaData/TopLeftLatitude'
},
{
self.Name_ID: 'topleftlongitude', # 左上角经度 必填
self.Name_XPath: '/ProductMetaData/TopLeftLongitude'
},
{
self.Name_ID: 'toprightlatitude', # 右上角维度 必填
self.Name_XPath: '/ProductMetaData/TopRightLatitude'
},
{
self.Name_ID: 'toprightlongitude', # 右上角经度 必填
self.Name_XPath: '/ProductMetaData/TopRightLongitude'
},
{
self.Name_ID: 'bottomrightlatitude', # 右下角维度 必填
self.Name_XPath: '/ProductMetaData/BottomRightLatitude'
},
{
self.Name_ID: 'bottomrightlongitude', # 右下角经度 必填
self.Name_XPath: '/ProductMetaData/BottomRightLongitude'
},
{
self.Name_ID: 'bottomleftlatitude', # 左下角维度 必填
self.Name_XPath: '/ProductMetaData/BottomLeftLatitude'
},
{
self.Name_ID: 'bottomleftlongitude', # 左下角经度 必填
self.Name_XPath: '/ProductMetaData/BottomLeftLongitude'
},
{
self.Name_ID: 'transformimg', # 斜视图,可空,不用质检
self.Name_Value: None
},
{
self.Name_ID: 'centertime', # 影像获取时间 必填
self.Name_XPath: '/ProductMetaData/CenterTime'
},
{
self.Name_ID: 'resolution', # 分辨率(米) 对应卫星的默认值，从info里取
self.Name_Custom_Item: {
'default': 2,
'Pan': '/ProductMetaData/ImageGSD'
}
},
{
self.Name_ID: 'rollangle', # 侧摆角
self.Name_XPath: '/ProductMetaData/RollViewingAngle'
},
{
self.Name_ID: 'cloudpercent', # 云量
self.Name_XPath: '/ProductMetaData/CloudPercent'
},
{
self.Name_ID: 'dataum', # 坐标系 默认为null
self.Name_Value: 'WGS_1984'
},
{
self.Name_ID: 'acquisition_id', # 轨道号
self.Name_XPath: '/ProductMetaData/OrbitID'
},
{
self.Name_ID:
'copyright', # 发布来源 从info取
self.Name_Value:
CUtils.dict_value_by_name(self.get_information(),
self.Plugins_Info_CopyRight, None)
},
{
self.Name_ID: 'publishdate', # 发布时间 必填
self.Name_XPath: '/ProductMetaData/ProduceTime',
self.Name_Value: None
},
{
self.Name_ID: 'remark', # 备注 可空
self.Name_Value: None
},
{
self.Name_ID: 'productname', # 产品名称，有的能从卫星元数据里面取，没有就不取
self.Name_XPath: None
},
{
self.Name_ID:
'producttype', # 产品类型 必填
self.Name_Value:
CUtils.dict_value_by_name(self.get_information(),
self.Plugins_Info_ProductType, None)
},
{
self.Name_ID: 'productattribute', # 产品属性 必填
self.Name_XPath: '/ProductMetaData/ProductLevel',
self.Name_Map: { # 映射，当取到的值为key时，将值转换为value
'LEVEL1A': 'L1',
'LEVEL2A': 'L2',
'LEVEL4A': 'L4'
# self.Name_Default: None # 没有对应的的映射使用默认值
}
},
{
self.Name_ID: 'productid', # 产品id 默认取主文件全名
self.Name_XPath: '/ProductMetaData/ProductID'
},
{
self.Name_ID: 'otherxml', # 预留字段，可空，配置正则
self.Name_Value: None
}
]
def process(self) -> str:
"""
在这里处理将__file_info__中记录的对象所对应的文件或目录信息, 根据tags_parser_rule的定义, 进行标签识别
:return:
"""
# 调用父类方法
super().process()
if not isinstance(self._tags_parser_rule, list):
return CResult.merge_result(self.Failure,
'标签解析规则必须是一个数组, 您的配置有误, 请检查!')
error_list = []
for tags_parser in self._tags_parser_rule:
catalog = CUtils.any_2_str(
CUtils.dict_value_by_name(tags_parser, self.Name_Catalog, ''))
tag_field_name = CUtils.any_2_str(
CUtils.dict_value_by_name(tags_parser, self.Name_Tag, ''))
keyword_field_list = CUtils.dict_value_by_name(
tags_parser, self.Name_Keyword, None)
data_sample = CUtils.any_2_str(
CUtils.dict_value_by_name(tags_parser, self.Name_Data_Sample,
''))
separator = CUtils.dict_value_by_name(tags_parser,
self.Name_Separator, None)
enable = CUtils.dict_value_by_name(tags_parser, self.Name_Enable,
True)
fuzzy_matching = CUtils.dict_value_by_name(
tags_parser, self.Name_Fuzzy_Matching, False)
if not enable:
continue
if CUtils.equal_ignore_case(tag_field_name, ''):
continue
if CUtils.equal_ignore_case(catalog, ''):
continue
if keyword_field_list is None:
continue
if len(keyword_field_list) == 0:
continue
if CUtils.equal_ignore_case(data_sample,
self.Tag_DataSample_MainName):
# 如果是主名, 则将对象名称和别名, 都以文件路径的格式, 补充在主名之后, 合并进行分类识别
tag_data_sample_str = CFile.join_file(
CFile.join_file(
CUtils.any_2_str(self.file_info.file_main_name),
self.object_name), self.__file_alias_name)
elif CUtils.equal_ignore_case(data_sample,
self.Tag_DataSample_RelationPath):
tag_data_sample_str = CUtils.any_2_str(
self.file_info.file_path_with_rel_path)
else:
# 如果是主名, 则将对象名称和别名, 都以文件路径的格式, 补充在相对路径的文件主名之后, 合并进行分类识别
tag_data_sample_str = CFile.join_file(
CFile.join_file(
CUtils.any_2_str(
self.file_info.file_main_name_with_rel_path),
self.object_name), self.__file_alias_name)
try:
tag_data_sample_list = CUtils.split(tag_data_sample_str,
separator)
self.process_tag(catalog, tag_field_name, keyword_field_list,
tag_data_sample_list, fuzzy_matching)
except Exception as error:
error_list.append(
'对象[{0}]在处理标签库[{1}]分类[{2}]有误, 详细错误信息为: {3}'.format(
self.object_name, catalog, tag_data_sample_str,
error.__str__()))
if len(error_list) == 0:
return CResult.merge_result(
self.Success, '文件或目录[{0}]对象业务分类解析成功完成!'.format(
self.file_info.file_main_name_with_rel_path))
else:
error_message = '文件或目录[{0}]的业务分类解析处理完毕, 但解析过程中出现了错误, 具体如下: \n'.format(
self.file_info.file_main_name_with_rel_path)
for error_str in error_list:
error_message = CUtils.str_append(error_message, error_str)
return CResult.merge_result(self.Success, error_message)
def get_metadata_bus_configuration_list(self) -> list:
"""
固定的列表，重写时不可缺项
"""
return [
{
self.Name_ID: 'satelliteid', # 卫星，必填，从元数据组织定义，必须是标准命名的卫星名称
self.Name_XPath: '/MetaData/ProductInfo/SatelliteID'
},
{
self.Name_ID: 'sensorid', # 传感器 必填,从元数据组织定义，必须是标准命名的传感器名称
self.Name_XPath: '/MetaData/ProductInfo/Sensor'
},
{
self.Name_ID: 'centerlatitude', # 中心维度
self.Name_XPath: '/MetaData/ProductInfo/CenterLatitude'
},
{
self.Name_ID: 'centerlongitude', # 中心经度
self.Name_XPath: '/MetaData/ProductInfo/CenterLongitude'
},
{
self.Name_ID: 'topleftlatitude', # 左上角维度 必填
self.Name_XPath: '/MetaData/ProductInfo/UpperLeftLatitude'
},
{
self.Name_ID: 'topleftlongitude', # 左上角经度 必填
self.Name_XPath: '/MetaData/ProductInfo/UpperLeftLongitude'
},
{
self.Name_ID: 'toprightlatitude', # 右上角维度 必填
self.Name_XPath: '/MetaData/ProductInfo/UpperRightLatitude'
},
{
self.Name_ID: 'toprightlongitude', # 右上角经度 必填
self.Name_XPath: '/MetaData/ProductInfo/UpperRightLongitude'
},
{
self.Name_ID: 'bottomrightlatitude', # 右下角维度 必填
self.Name_XPath: '/MetaData/ProductInfo/LowerRightLatitude'
},
{
self.Name_ID: 'bottomrightlongitude', # 右下角经度 必填
self.Name_XPath: '/MetaData/ProductInfo/LowerRightLongitude'
},
{
self.Name_ID: 'bottomleftlatitude', # 左下角维度 必填
self.Name_XPath: '/MetaData/ProductInfo/LowerLeftLatitude'
},
{
self.Name_ID: 'bottomleftlongitude', # 左下角经度 必填
self.Name_XPath: '/MetaData/ProductInfo/LowerLeftLongitude'
},
{
self.Name_ID: 'transformimg', # 斜视图,可空,不用质检
self.Name_Value: None
},
{
self.Name_ID: 'centertime', # 影像获取时间 必填
self.Name_XPath: '/MetaData/ProductInfo/StartTime'
},
{
self.Name_ID: 'resolution', # 分辨率(米) 对应卫星的默认值，从info里取
self.Name_XPath: '/MetaData/ProductInfo/ImageRowGSD'
},
{
self.Name_ID: 'rollangle', # 侧摆角
self.Name_XPath: '/MetaData/ProductInfo/RollSatelliteAngle'
},
{
self.Name_ID: 'cloudpercent', # 云量
self.Name_XPath: '/MetaData/ProductInfo/CloudPercent'
},
{
self.Name_ID: 'dataum', # 坐标系 默认为null
self.Name_Value: 'WGS_1984'
},
{
self.Name_ID: 'acquisition_id', # 轨道号
self.Name_XPath: '/MetaData/ProductInfo/OrbitID'
},
{
self.Name_ID: 'copyright', # 发布来源 从info取
self.Name_Value: CUtils.dict_value_by_name(self.get_information(), self.Plugins_Info_CopyRight, None)
},
{
self.Name_ID: 'publishdate', # 发布时间 必填
self.Name_XPath: '/MetaData/ProductInfo/EndTime'
},
{
self.Name_ID: 'remark', # 备注 可空
self.Name_Value: None
},
{
self.Name_ID: 'productname', # 产品名称，有的能从卫星元数据里面取，没有就不取
self.Name_XPath: '/MetaData/ProductInfo/SceneID'
},
{
self.Name_ID: 'producttype', # 产品类型 必填
self.Name_Value: CUtils.dict_value_by_name(self.get_information(), self.Plugins_Info_ProductType, None)
},
{
self.Name_ID: 'productattribute', # 产品属性 必填
self.Name_XPath: '/MetaData/ProductInfo/ProductLevel'
},
{
self.Name_ID: 'productid', # 产品id 默认取主文件全名
self.Name_XPath: '/MetaData/ProductInfo/ProductID'
},
{
self.Name_ID: 'otherxml', # 预留字段，可空，配置正则
self.Name_Value: '(?i).*MSS.*_meta.xml'
}
]
def parser_metadata_custom(self, parser: CMetaDataParser) -> str:
"""
自定义的元数据解析, 在所有质检和其他处理之后触发
:param parser:
:return:
"""
meta_data_json = parser.metadata.metadata_json()
if meta_data_json is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 无法获取JSON格式的元数据! '.format(
self.file_info.file_name_with_full_path,
)
)
json_data_source = meta_data_json.xpath_one('datasource', None)
layer_list = meta_data_json.xpath_one(self.Name_Layers, None)
if layer_list is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 元数据中无法找到layers节点! '.format(
self.file_info.file_name_with_full_path,
)
)
mdb_ib_id = CFactory().give_me_db(self.file_info.db_server_id).one_value(
'''
select dso_ib_id
from dm2_storage_object
where dsoid = :object_id
''',
{
'object_id': parser.object_id
}
)
error_message_list = []
table = CTable()
table.load_info(self.file_info.db_server_id, self.TableName_DM_Object)
for layer in layer_list:
layer_name = CUtils.dict_value_by_name(layer, self.Name_Name, '')
if CUtils.equal_ignore_case(layer_name, ''):
continue
layer_alias_name = CUtils.dict_value_by_name(layer, self.Name_Description, layer_name)
layer_metadata_json = CJson()
layer_metadata_json.set_value_of_name('datasource', json_data_source)
layer_metadata_json.set_value_of_name('layer_count', 1)
layer_metadata_json.set_value_of_name('layers', [layer])
layer_metadata_text = layer_metadata_json.to_json()
try:
sql_find_layer_existed = '''
select dsoid as layer_id_existed
from dm2_storage_object
where upper(dsoobjectname) = upper(:layer_name)
and dsoparentobjid = :object_id
'''
layer_id_existed = CFactory().give_me_db(self.file_info.db_server_id).one_value(
sql_find_layer_existed,
{
'layer_name': layer_name,
'object_id': parser.object_id
}
)
if layer_id_existed is None:
layer_id_existed = CUtils.one_id()
table.column_list.reset()
table.column_list.column_by_name('dsoid').set_value(layer_id_existed)
table.column_list.column_by_name('dsoobjectname').set_value(layer_name)
table.column_list.column_by_name('dsoobjecttype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Plugins_Name,
''
)
)
table.column_list.column_by_name('dsodatatype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Data_Type,
''
)
)
table.column_list.column_by_name('dsoalphacode').set_value(CUtils.alpha_text(layer_name))
table.column_list.column_by_name('dsoaliasname').set_value(layer_alias_name)
table.column_list.column_by_name('dsoparentobjid').set_value(parser.object_id)
table.column_list.column_by_name('dso_ib_id').set_value(mdb_ib_id)
table.column_list.column_by_name('dsometadatatext').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadatajson').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadataparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsotagsparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsodetailparsestatus').set_value(self.ProcStatus_InQueue)
result = table.save_data()
if not CResult.result_success(result):
error_message_list.append(
'图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(
layer_name,
CResult.result_message(result)
)
)
except Exception as error:
error_message_list.append('图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(layer_name, error.__str__()))
if len(error_message_list) > 0:
return CResult.merge_result(
self.Failure,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 详细情况如下: \n{1}'.format(
self.file_info.file_name_with_full_path,
CUtils.list_2_str(error_message_list, '', '\n', '', True)
)
)
else:
return CResult.merge_result(
self.Success,
'数据[{0}]的自定义元数据解析完毕! '.format(
self.file_info.file_name_with_full_path,
)
)
def get_metadata_bus_configuration_list(self) -> list:
"""
固定的列表，重写时不可缺项
"""
return [
{
self.Name_ID: 'satelliteid', # 卫星，必填，从元数据组织定义，必须是标准命名的卫星名称
self.Name_XPath: '/product/sourceAttributes/satellite'
},
{
self.Name_ID: 'sensorid', # 传感器 必填,从元数据组织定义，必须是标准命名的传感器名称
self.Name_XPath: '/product/sourceAttributes/sensor'
},
{
self.Name_ID: 'centerlatitude', # 中心维度
self.Name_XPath: None
},
{
self.Name_ID: 'centerlongitude', # 中心经度
self.Name_XPath: None
},
{
self.Name_ID: 'topleftlatitude', # 左上角维度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'topleftlongitude', # 左上角经度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'toprightlatitude', # 右上角维度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'toprightlongitude', # 右上角经度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'bottomrightlatitude', # 右下角维度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'bottomrightlongitude', # 右下角经度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'bottomleftlatitude', # 左下角维度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'bottomleftlongitude', # 左下角经度 必填
self.Name_XPath: None
},
{
self.Name_ID: 'transformimg', # 斜视图,可空,不用质检
self.Name_Value: None
},
{
self.Name_ID: 'centertime', # 影像获取时间 必填
self.Name_XPath: '/product/sourceAttributes/rawDataStartTime'
},
{
self.Name_ID:
'resolution', # 分辨率(米) 对应卫星的默认值，从info里取
self.Name_XPath:
'/product/imageAttributes/rasterAttributes/sampledPixelSpacing'
},
{
self.Name_ID:
'rollangle', # 侧摆角
self.Name_XPath:
'/product/sourceAttributes/orbitAndAttitude/attitudeInformation/attitudeAngles/roll'
},
{
self.Name_ID: 'cloudpercent', # 云量
self.Name_Value: 0
},
{
self.Name_ID: 'dataum', # 坐标系 默认为null
self.Name_Value: 'WGS_1984'
},
{
self.Name_ID: 'acquisition_id', # 轨道号
self.Name_XPath: None
},
{
self.Name_ID:
'copyright', # 发布来源 从info取
self.Name_Value:
CUtils.dict_value_by_name(self.get_information(),
self.Plugins_Info_CopyRight, None)
},
{
self.Name_ID:
'publishdate', # 发布时间 必填
self.Name_XPath:
'/product/imageGenerationParameters/generalProcessingInformation/processingTime'
},
{
self.Name_ID: 'remark', # 备注 可空
self.Name_Value: None
},
{
self.Name_ID: 'productname', # 产品名称，有的能从卫星元数据里面取，没有就不取
self.Name_XPath: None
},
{
self.Name_ID:
'producttype', # 产品类型 必填
self.Name_Value:
CUtils.dict_value_by_name(self.get_information(),
self.Plugins_Info_ProductType, None)
},
{
self.Name_ID: 'productattribute', # 产品属性 必填
self.Name_Value: 'L1'
},
{
self.Name_ID: 'productid', # 产品id 默认取主文件全名
self.Name_XPath: '/product/productId'
},
{
self.Name_ID: 'otherxml', # 预留字段，可空，配置正则
self.Name_Value: None
}
]
def get_metadata_bus_configuration_list(self) -> list:
"""
固定的列表，重写时不可缺项
"""
return [
{
self.Name_ID:
'satelliteid', # 卫星，必填，从元数据组织定义，必须是标准命名的卫星名称
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="SPACECRAFT_ID"]'
},
{
self.Name_ID:
'sensorid', # 传感器 必填,从元数据组织定义，必须是标准命名的传感器名称
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="SENSOR_ID"]'
},
{
self.Name_ID: 'centerlatitude', # 中心维度
self.Name_XPath: None
},
{
self.Name_ID: 'centerlongitude', # 中心经度
self.Name_XPath: None
},
{
self.Name_ID:
'topleftlatitude', # 左上角维度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_UL_LAT_PRODUCT"]'
},
{
self.Name_ID:
'topleftlongitude', # 左上角经度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_UL_LON_PRODUCT"]'
},
{
self.Name_ID:
'toprightlatitude', # 右上角维度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_UR_LAT_PRODUCT"]'
},
{
self.Name_ID:
'toprightlongitude', # 右上角经度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_UR_LON_PRODUCT"]'
},
{
self.Name_ID:
'bottomrightlatitude', # 右下角维度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_LL_LAT_PRODUCT"]'
},
{
self.Name_ID:
'bottomrightlongitude', # 右下角经度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_LL_LON_PRODUCT"]'
},
{
self.Name_ID:
'bottomleftlatitude', # 左下角维度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_LR_LAT_PRODUCT"]'
},
{
self.Name_ID:
'bottomleftlongitude', # 左下角经度 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="CORNER_LR_LON_PRODUCT"]'
},
{
self.Name_ID: 'transformimg', # 斜视图,可空,不用质检
self.Name_Value: None
},
{
self.Name_ID: 'centertime', # 影像获取时间 必填
self.Name_Custom_Item: {
self.Name_Time_Date:
'//item[@name="PRODUCT_METADATA"]//item[@name="DATE_ACQUIRED"]',
self.Name_Time_Time:
'//item[@name="PRODUCT_METADATA"]//item[@name="SCENE_CENTER_TIME"]'
}
},
{
self.Name_ID:
'resolution', # 分辨率(米) 对应卫星的默认值，从info里取
self.Name_XPath:
'//item[@name="PROJECTION_PARAMETERS"]//item[@name="GRID_CELL_SIZE_PANCHROMATIC"]'
},
{
self.Name_ID:
'rollangle', # 侧摆角
self.Name_XPath:
'//item[@name="IMAGE_ATTRIBUTES"]//item[@name="ROLL_ANGLE"]'
},
{
self.Name_ID:
'cloudpercent', # 云量
self.Name_XPath:
'//item[@name="IMAGE_ATTRIBUTES"]//item[@name="CLOUD_COVER"]'
},
{
self.Name_ID: 'dataum', # 坐标系 默认为null
self.Name_Value: 'WGS_1984'
},
{
self.Name_ID: 'acquisition_id', # 轨道号
self.Name_Value: None
},
{
self.Name_ID:
'copyright', # 发布来源 从info取
self.Name_XPath:
'//item[@name="METADATA_FILE_INFO"]//item[@name="ORIGIN"]'
},
{
self.Name_ID:
'publishdate', # 发布时间 必填
self.Name_XPath:
'//item[@name="METADATA_FILE_INFO"]//item[@name="FILE_DATE"]'
},
{
self.Name_ID: 'remark', # 备注 可空
self.Name_Value: None
},
{
self.Name_ID: 'productname', # 产品名称，有的能从卫星元数据里面取，没有就不取
self.Name_XPath: None
},
{
self.Name_ID:
'producttype', # 产品类型 必填
self.Name_Value:
CUtils.dict_value_by_name(self.get_information(),
self.Plugins_Info_ProductType, None)
},
{
self.Name_ID:
'productattribute', # 产品属性 必填
self.Name_XPath:
'//item[@name="PRODUCT_METADATA"]//item[@name="DATA_TYPE"]'
},
{
self.Name_ID:
'productid', # 产品id 默认取主文件全名
self.Name_XPath:
'//item[@name="METADATA_FILE_INFO"]//item[@name="LANDSAT_SCENE_ID"]'
},
{
self.Name_ID: 'otherxml', # 预留字段，可空，配置正则
self.Name_Value: None
}
]