def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 王学谦
:param parser:
:return:
"""
super().qa_file_custom(parser)
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
check_file_metadata_bus_exist = False
ext = self.Transformer_XML
metadata_name_with_path = CFile.join_file(
file_path, '{0}M.xml'.format(file_main_name[:-1]))
if CFile.file_or_path_exist(metadata_name_with_path):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = metadata_name_with_path
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
'',
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Error,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'本文件缺少业务元数据'
})
else:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
self.metadata_bus_src_filename_with_path,
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Pass,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'业务元数据[{0}]存在'.format(
CFile.file_name(self.metadata_bus_src_filename_with_path))
})
def classified(self):
"""
设计国土行业数据的dom_10_dom验证规则
完成 负责人 李宪 在这里检验dom_10_dom的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
check_file_main_name_length = len(file_main_name) == 13
if not check_file_main_name_length:
return self.Object_Confirm_IUnKnown, self._object_name
file_main_name_with_path = CFile.join_file(self.file_info.file_path, file_main_name)
check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
if not check_file_main_name_exist:
return self.Object_Confirm_IUnKnown, self._object_name
"""
下面判别第1位是字母
下面判别第4位是字母
下面判别第23位是数字
下面判别第567位是数字
下面判别第8910位是数字
下面判别第111213位是DOM
"""
char_1 = file_main_name[0:1]
char_2_3 = file_main_name[1:3]
char_4 = file_main_name[3:4]
char_5_to_7 = file_main_name[4:7]
char_8_to_10 = file_main_name[7:10]
char_11_to_13 = file_main_name[10:13]
if CUtils.text_is_alpha(char_1) is False \
or CUtils.text_is_numeric(char_2_3) is False \
or CUtils.text_is_alpha(char_4) is False \
or CUtils.text_is_numeric(char_5_to_7) is False \
or CUtils.text_is_numeric(char_8_to_10) is False \
or CUtils.equal_ignore_case(char_11_to_13, "DOM") is False:
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def file2param(cls, params: dict, param_name: str, file_name: str):
if params is None:
return
if not CFile.file_or_path_exist(file_name):
params[param_name] = None
return
# 注意这里一定要使用rb，读出二进制文件，否则有读不全等问题
fp = open(file_name, 'rb')
try:
params[param_name] = fp.read()
finally:
fp.close()
def classified(self):
"""
设计国土行业数据custom的验证规则（自定义影像）
完成 负责人 李宪 在这里检验custom的识别规则
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
file_object_name = file_main_name
file_name_with_full_path = self.file_info.file_name_with_full_path # 初始化需要的参数
if file_name_with_full_path.endswith('_21at.xml'):
file_object_name = file_main_name[:-5]
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_object_name)
check_file_main_name_exist_tif = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Tif))
check_file_main_name_exist_img = CFile.file_or_path_exist(
'{0}.{1}'.format(file_main_name_with_path, self.Name_Img))
if (not check_file_main_name_exist_tif) and (
not check_file_main_name_exist_img):
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif) \
or CUtils.equal_ignore_case(file_ext, self.Name_Img):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = self.file_info.file_main_name
file_detail_xml = '{0}_21at.xml'.format(
self.file_info.file_main_name_with_full_path)
self.add_file_to_details(file_detail_xml) # 将文件加入到附属文件列表中
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def process_mission(self, dataset) -> str:
"""
:param dataset:
:return:
"""
ds_na_id = dataset.value_by_name(0, 'na_id', '')
ds_app_id = dataset.value_by_name(0, 'app_id', '')
ds_object_id = dataset.value_by_name(0, 'object_id', '')
ds_object_type = dataset.value_by_name(0, 'object_type', '')
ds_object_name = dataset.value_by_name(0, 'object_name', '')
ds_object_access = dataset.value_by_name(0, 'object_access',
self.DataAccess_Forbid)
CLogger().debug('与第三方模块[{0}]同步的对象为: [{1}]'.format(
ds_app_id, ds_object_name))
try:
module_file_name = CFile.join_file(
CSys.get_metadata_data_access_modules_root_dir(),
'{0}.{1}'.format(ds_app_id, self.FileExt_Py))
if not CFile.file_or_path_exist(module_file_name):
message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id)
result = CResult.merge_result(self.Success, message)
self.update_sync_result(ds_na_id, result)
return result
module_obj = CObject.create_module_instance(
CSys.get_metadata_data_access_modules_root_name(), ds_app_id,
self.get_mission_db_id())
if module_obj is None:
message = '第三方模块[{0}]没有设置对应的算法, 直接通过!'.format(ds_app_id)
result = CResult.merge_result(self.Success, message)
self.update_sync_result(ds_na_id, result)
return result
module_title = CUtils.dict_value_by_name(module_obj.information(),
self.Name_Title, '')
result = module_obj.sync(ds_object_access, ds_object_id,
ds_object_name, ds_object_type, None)
self.update_sync_result(ds_na_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure,
'与第三方模块[{0}]同步的对象: [{1}]的同步过程出现异常, 详细情况: [{2}]!'.format(
ds_app_id, ds_object_name, error.__str__()))
self.update_sync_result(ds_na_id, result)
return result
def classified(self):
"""
设计国土行业数据的dom-10验证规则
完成 负责人 李宪 在这里检验dom-10的元数据文件格式时, 应该一个一个类型的对比, 找到文件时, 将该文件的格式和文件名存储到类的私有属性中, 以便在元数据处理时直接使用
:return:
"""
super().classified()
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext
check_file_main_name_length = len(file_main_name) == 10
if not check_file_main_name_length:
return self.Object_Confirm_IUnKnown, self._object_name
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_main_name)
check_file_main_name_exist = CFile.file_or_path_exist('{0}.{1}'.format(
file_main_name_with_path, self.Name_Tif))
if not check_file_main_name_exist:
return self.Object_Confirm_IUnKnown, self._object_name
"""
下面判别第1位是字母
下面判别第4位是字母
下面判别第23位是数字
下面判别第567位是数字
下面判别第8910位是数字
"""
char_1 = file_main_name[0:1]
char_2_3 = file_main_name[1:3]
char_4 = file_main_name[3:4]
char_5_to_7 = file_main_name[4:7]
char_8_to_10 = file_main_name[7:10]
if CUtils.text_is_alpha(char_1) is False \
or CUtils.text_is_numeric(char_2_3) is False \
or CUtils.text_is_alpha(char_4) is False \
or CUtils.text_is_numeric(char_5_to_7) is False \
or CUtils.text_is_numeric(char_8_to_10) is False:
return self.Object_Confirm_IUnKnown, self._object_name
if CUtils.equal_ignore_case(file_ext, self.Name_Tif):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
return self._object_confirm, self._object_name
def create_virtual_content(self) -> bool:
"""
:return:
"""
zip_obj = CZip(self.__target_name__)
try:
zip_obj.open()
zip_obj.extract_all(self.__virtual_content_root_dir__)
except:
return False
finally:
zip_obj.close()
return super().create_virtual_content() and CFile.file_or_path_exist(
self.__virtual_content_root_dir__)
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 王学谦
:param parser:
:return:
"""
super().qa_file_custom(parser)
metadata_main_name_with_path = CFile.join_file(
self.file_info.file_path, self.file_info.file_main_name)
check_file_metadata_bus_exist = False
temp_metadata_bus_file = '{0}.xml'.format(
metadata_main_name_with_path[:-2])
if CFile.file_or_path_exist(temp_metadata_bus_file):
check_file_metadata_bus_exist = True
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
'',
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Error,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'本文件缺少业务元数据'
})
else:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
self.metadata_bus_src_filename_with_path,
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Pass,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path)
})
def is_raster_file_integrity(cls, raster_file_with_path: str) -> bool:
"""
判断影像数据的文件完整性，img
xxx.img 文件 栅格数据可读 错误
xxx.ige 文件 img小于1M时必须存在 警告
@param raster_file_with_path:
@return:
"""
file_ext = CFile.file_ext(raster_file_with_path)
if file_ext.lower() == 'img':
size = CFile.file_size(raster_file_with_path)
if size < 1024 * 1024:
file_main_name = CFile.file_main_name(raster_file_with_path)
file_path = CFile.file_path(raster_file_with_path)
ige = CFile.join_file(file_path, file_main_name + '.ige')
if not CFile.file_or_path_exist(ige):
return False
return True
def classified(self):
file_main_name = self.file_info.file_main_name
file_ext = self.file_info.file_ext # 初始化需要的参数
file_object_name = file_main_name[:]
file_main_name_with_path = CFile.join_file(self.file_info.file_path,
file_object_name)
if CUtils.equal_ignore_case(file_ext, self.Name_Shp):
self._object_confirm = self.Object_Confirm_IKnown
self._object_name = file_main_name
else:
if CFile.file_or_path_exist('{0}.{1}'.format(
file_main_name_with_path, self.Name_Shp)):
self._object_confirm = self.Object_Confirm_IKnown_Not
self._object_name = None
else:
self._object_confirm = self.Object_Confirm_IUnKnown
self._object_name = None
return self._object_confirm, self._object_name
def plugins(cls, file_info: CDMFilePathInfoEx,
plugins_id: str) -> CPlugins:
"""
根据给定的文件信息和插件名称, 直接创建插件对象
如果插件文件不存在, 则返回None
:param file_info:
:param plugins_id:
:return:
"""
target_type = file_info.file_type
target_path = CFile.join_file(CSys.get_plugins_root_dir(), target_type)
plugins_file_name = CFile.join_file(
target_path, '{0}.{1}'.format(plugins_id, cls.FileExt_Py))
if not CFile.file_or_path_exist(plugins_file_name):
return None
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), target_type)
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, plugins_id, file_info)
return class_classified_obj
def __process_file_format__(self, metadata_format: str):
if not CFile.file_or_path_exist(self.transformer_src_filename):
return CResult.merge_result(
self.Failure,
'文件[{0}]不存在, 无法解析! '.format(self.transformer_src_filename))
try:
self.__metadata__.set_metadata_bus_file(
self.Success,
'文件[{0}]成功加载! '.format(self.transformer_src_filename),
metadata_format, self.transformer_src_filename)
return CResult.merge_result(
self.Success,
'文件[{0}]成功加载! '.format(self.transformer_src_filename))
except:
self.__metadata__.set_metadata_bus(
self.Exception, '元数据文件[{0}]格式不合法, 无法处理! '.format(
self.transformer_src_filename), self.MetaDataFormat_Text,
'')
return CResult.merge_result(
self.Exception, '元数据文件[{0}]格式不合法, 无法处理! '.format(
self.transformer_src_filename))
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 李宪
:param parser:
:return:
"""
metadata_main_name_with_path = CFile.join_file(self.file_info.file_path, self.file_info.file_main_name)
check_file_metadata_bus_exist = False
ext = self.Transformer_XML
temp_metadata_bus_file = '{0}_21at.xml'.format(metadata_main_name_with_path)
if CFile.file_or_path_exist(temp_metadata_bus_file):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = temp_metadata_bus_file
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本文件缺少业务元数据，是正常现象'
}
)
else:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: self.metadata_bus_src_filename_with_path,
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path)
}
)
def open(self, encoding: str = 'UTF-8'):
gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "YES")
# gdal.SetConfigOption("SHAPE_ENCODING", "GBK")
gdal.SetConfigOption("SHAPE_ENCODING", encoding)
# 注册所有的驱动
ogr.RegisterAll()
self._active = False
if not CFile.file_or_path_exist(self.data_file_or_path):
raise FileNotFoundError(self.data_file_or_path)
self._data_obj = ogr.Open(self.data_file_or_path, 0)
mdb_flag = False
if self._data_obj is None:
self._data_obj = ogr.Open(
self.data_file_or_path.encode(CResource.Encoding_GBK), 0)
mdb_flag = True
if self._data_obj is None:
raise FileCanNotOpenException(self.data_file_or_path)
self._active = True
self._parser_datasets(mdb_flag)
def process_mission(self, dataset) -> str:
"""
详细算法复杂, 参见readme.md中[### 数据入库调度]章节
:param dataset:
:return:
"""
ds_src_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_src_storage_type = dataset.value_by_name(0, 'query_storage_type',
self.Storage_Type_Mix)
ds_src_root_path = dataset.value_by_name(0, 'query_rootpath', '')
ds_src_dir_id = dataset.value_by_name(0, 'query_ib_dir_id', '')
ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '')
ds_ib_directory_name = dataset.value_by_name(0,
'query_ib_relation_dir',
'')
ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '')
ds_ib_option = dataset.value_by_name(0, 'query_ib_option', '')
src_need_storage_size = self.get_storage_size(ds_ib_id,
ds_src_storage_id,
ds_ib_directory_name,
ds_ib_option)
src_path = ds_src_root_path
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
src_path = CFile.join_file(src_path, ds_ib_directory_name)
src_dataset_metadata_filename = CFile.join_file(
src_path, self.FileName_MetaData_Bus_21AT)
CLogger().debug('入库的目录为: {0}.{1}'.format(ds_ib_id,
ds_ib_directory_name))
try:
# 检查所有文件与元数据是否相符
all_ib_file_or_path_existed = self.check_all_ib_file_or_path_existed(
ds_ib_id)
if not CResult.result_success(all_ib_file_or_path_existed):
self.update_ib_result(ds_ib_id, all_ib_file_or_path_existed)
return all_ib_file_or_path_existed
# 将数据入库的记录保存到日志中
result = self.ib_log(ds_ib_id, ds_src_storage_id,
ds_ib_directory_name)
if not CResult.result_success(result):
self.update_ib_result(ds_ib_id, result)
return result
# 如果是在核心存储或混合存储中直接入库, 则仅仅改变元数据状态即可
if CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Mix) \
or CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Core):
result_ib_in_core_or_mix_storage = self.update_ib_data_status_in_core_or_mix_storage(
ds_ib_id, ds_src_storage_id, ds_ib_directory_name,
ds_src_dir_id)
self.update_ib_result(ds_ib_id,
result_ib_in_core_or_mix_storage)
return result_ib_in_core_or_mix_storage
# 加载目录下的待入库数据集的元数据文件
src_dataset_xml = CXml()
src_dataset_type = self.Name_Default
if CFile.file_or_path_exist(src_dataset_metadata_filename):
src_dataset_xml.load_file(src_dataset_metadata_filename)
src_dataset_type = CXml.get_element_text(
src_dataset_xml.xpath_one(self.Path_MD_Bus_ProductType))
if CUtils.equal_ignore_case(src_dataset_type, ''):
src_dataset_type = self.Name_Default
# 获取匹配的入库模式
src_ib_schema = self.get_ib_schema(src_dataset_type, ds_ib_option)
if src_ib_schema is None:
result = CResult.merge_result(
self.Failure,
'目录为[{0}.{1}]的数据集类型为[{2}], 未找到匹配的入库模式, 请检查修正后重试!'.format(
ds_ib_id, ds_ib_directory_name, src_dataset_type))
self.update_ib_result(ds_ib_id, result)
return result
# 计算入库的目标存储\存储根目录\目标子目录在目标存储中的副目录的标识\目标子目录\反馈消息
dest_ib_storage_id, dest_ib_root_path, desc_ib_dir_id, dest_ib_subpath, message = self.get_dest_storage(
ds_ib_batch_no, src_need_storage_size, ds_ib_option,
src_ib_schema, src_dataset_xml)
if dest_ib_storage_id is None or dest_ib_subpath is None:
result = CResult.merge_result(self.Failure, message)
self.update_ib_result(ds_ib_id, result)
return result
dest_ib_subpath = CFile.unify(dest_ib_subpath)
if CJson.json_attr_value(ds_ib_option,
self.Path_IB_Switch_CheckFileLocked,
self.DB_False) == self.DB_True:
src_ib_files_not_locked, message = self.check_src_ib_files_not_locked(
ds_src_root_path, src_path)
if not src_ib_files_not_locked:
result = CResult.merge_result(self.Failure, message)
self.update_ib_result(ds_ib_id, result)
return result
proc_ib_src_path = ds_src_root_path
proc_ib_dest_path = dest_ib_root_path
if not CUtils.equal_ignore_case(dest_ib_subpath, ''):
proc_ib_dest_path = CFile.join_file(dest_ib_root_path,
dest_ib_subpath)
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
proc_ib_src_path = CFile.join_file(proc_ib_src_path,
ds_ib_directory_name)
proc_ib_dest_path = CFile.join_file(proc_ib_dest_path,
ds_ib_directory_name)
# --------------------------------------------------------------至此, 数据入库前的检查处理完毕
# 移动源目录至目标目录, 如果是根目录, 则仅仅移动文件
result = self.ib_files_move(
proc_ib_src_path, proc_ib_dest_path,
CUtils.equal_ignore_case(ds_ib_directory_name, ''))
if not CResult.result_success(result):
# 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功
sub_result = self.ib_files_move(
proc_ib_dest_path, proc_ib_src_path,
CUtils.equal_ignore_case(ds_ib_directory_name, ''))
if not CResult.result_success(sub_result):
sub_result_message = CResult.result_message(sub_result)
result_message = CResult.result_message(result)
result = CResult.merge_result(
self.Failure,
'{0}\n{1}'.format(result_message, sub_result_message))
self.update_ib_result(ds_ib_id, result)
return result
# 将源文件的元数据, 移动至目标存储下, 如果出现异常, 则在方法内部rollback
result = self.src_ib_metadata_move_to_storage(
ds_ib_id, ds_src_storage_id, ds_src_dir_id,
ds_ib_directory_name, dest_ib_storage_id, desc_ib_dir_id,
dest_ib_subpath)
if not CResult.result_success(result):
# 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功
sub_result = self.ib_files_move(
proc_ib_dest_path, proc_ib_src_path,
CUtils.equal_ignore_case(ds_ib_directory_name, ''))
if not CResult.result_success(sub_result):
sub_result_message = CResult.result_message(sub_result)
result_message = CResult.result_message(result)
result = CResult.merge_result(
self.Failure,
'{0}/n{1}'.format(result_message, sub_result_message))
self.update_ib_result(ds_ib_id, result)
return result
result = CResult.merge_result(
self.Success,
'目录为[{0}.{1}]入库成功!'.format(ds_ib_id, ds_ib_directory_name))
self.update_ib_result(ds_ib_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure, '目录为[{0}.{1}]入库出现异常! 错误原因为: {2}'.format(
ds_ib_id, ds_ib_directory_name, error.__str__()))
self.update_ib_result(ds_ib_id, result)
return result
def process(self) -> str:
"""
在这里提取文档数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式
注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!!
:return:
"""
default_result = super().process()
out_metadata_file_fullname = CFile.join_file(
self.file_content.work_root_dir, self.FileName_MetaData)
in_file_fullname = self.file_info.file_name_with_full_path
if not settings.application.xpath_one(
self.Path_Setting_Dependence_Tika_Enable, True):
return default_result
tika_dependence_mode = settings.application.xpath_one(
self.Path_Setting_Dependence_Tika_Mode, self.Name_Server)
if CUtils.equal_ignore_case(tika_dependence_mode, self.Name_Server):
tika_server_url = settings.application.xpath_one(
self.Path_Setting_Dependence_Tika_Server_Url, None)
tika_server_connect_timeout = settings.application.xpath_one(
self.Path_Setting_Dependence_Tika_Server_Timeout, 30)
if CUtils.equal_ignore_case(tika_server_url, ''):
return default_result
try:
parsed = TikaServer.from_file(
in_file_fullname,
tika_server_url,
requestOptions={'timeout': tika_server_connect_timeout})
meta_data_dict = parsed["metadata"]
json_obj = CJson()
json_obj.load_obj(meta_data_dict)
json_obj.to_file(out_metadata_file_fullname)
return CResult.merge_result_info(
CResult.merge_result(
self.Success,
'文档[{0}]的元数据提取成功'.format(in_file_fullname)),
self.Name_Format, self.MetaDataFormat_Json)
except Exception as error:
return CResult.merge_result(
self.Failure, '文档[{0}]的元数据提取过程出现错误, 详细信息为: [{1}]'.format(
in_file_fullname, error.__str__()))
else:
tika_application = settings.application.xpath_one(
self.Path_Setting_Dependence_Tika_Client_App, None)
if CUtils.equal_ignore_case(tika_application, ''):
return default_result
if not CFile.file_or_path_exist(tika_application):
return CResult.merge_result(
self.Failure,
'文档[{0}]的元数据无法提取, 详细原因为: [依赖中间件{1}文件不存在, 请修正后重试!]'.format(
in_file_fullname, tika_application))
try:
tika_client = TikaApplication(file_jar=tika_application)
meta_data_dict = tika_client.extract_only_metadata(
in_file_fullname)
json_obj = CJson()
json_obj.load_obj(meta_data_dict)
json_obj.to_file(out_metadata_file_fullname)
return CResult.merge_result_info(
CResult.merge_result(
self.Success,
'文档[{0}]的元数据提取成功'.format(in_file_fullname)),
self.Name_Format, self.MetaDataFormat_Json)
except Exception as error:
return CResult.merge_result(
self.Failure, '文档[{0}]的元数据提取过程出现错误, 详细信息为: [{1}]'.format(
in_file_fullname, error.__str__()))
# result = raster_mdreader.get_metadata_2_file(out_metadata_file_fullname)
# result = CProcessUtils.processing_method(raster_mdreader.get_metadata_2_file, out_metadata_file_fullname)
# 进程调用模式
# p_one = Process(target=raster_mdreader.get_metadata_2_file, args=(out_metadata_file_fullname,))
# p_one.start()
# p_one.join()
return CResult.merge_result_info(result, self.Name_Format,
self.MetaDataFormat_Json)
def save_metadata_spatial(self) -> str:
"""
完成空间元数据的入库更新操作
:return:
"""
mdt_spatial_result, mdt_spatial_memo, mdt_spatial = self.metadata.metadata_spatial()
params = {
'dsoid': self.object_id,
'dso_spatial_result': mdt_spatial_result,
'dso_spatial_parsermemo': mdt_spatial_memo,
'dso_prj_wkt': mdt_spatial.prj_wkt,
'dso_prj_proj4': mdt_spatial.prj_proj4,
'dso_prj_project': mdt_spatial.prj_project,
'dso_prj_coordinate': mdt_spatial.prj_coordinate,
'dso_prj_degree': mdt_spatial.prj_degree,
'dso_prj_zone': mdt_spatial.prj_zone,
'dso_prj_source': mdt_spatial.prj_source
}
database = CFactory().give_me_db(self.file_info.db_server_id)
params['dso_center_native'] = CFile.file_2_str(mdt_spatial.native_center)
params['dso_geo_bb_native'] = CFile.file_2_str(mdt_spatial.native_box)
params['dso_geo_native'] = CFile.file_2_str(mdt_spatial.native_geom)
if CFile.file_or_path_exist(mdt_spatial.wgs84_center):
dso_center_wgs84 = database.sql.func_wkt2geometry(
CUtils.quote(CFile.file_2_str(mdt_spatial.wgs84_center)), self.SRID_WGS84)
else:
dso_center_wgs84 = 'null'
if CFile.file_or_path_exist(mdt_spatial.wgs84_bbox):
dso_geo_bb_wgs84 = database.sql.func_wkt2geometry(
CUtils.quote(
CFile.file_2_str(mdt_spatial.wgs84_bbox)), self.SRID_WGS84)
else:
dso_geo_bb_wgs84 = 'null'
if CFile.file_or_path_exist(mdt_spatial.wgs84_geom):
dso_geo_wgs84 = database.sql.func_wkt2geometry(
CUtils.quote(
CFile.file_2_str(mdt_spatial.wgs84_geom)), self.SRID_WGS84)
else:
dso_geo_wgs84 = 'null'
# 所有元数据入库
CFactory().give_me_db(self.file_info.db_server_id).execute(
'''
update dm2_storage_object
set dso_spatial_result = :dso_spatial_result
, dso_spatial_parsermemo = :dso_spatial_parsermemo
, dso_center_native = :dso_center_native
, dso_geo_bb_native = :dso_geo_bb_native
, dso_geo_native = :dso_geo_native
, dso_center_wgs84 = {0}
, dso_geo_bb_wgs84 = {1}
, dso_geo_wgs84 = {2}
, dso_prj_wkt = :dso_prj_wkt
, dso_prj_proj4 = :dso_prj_proj4
, dso_prj_project = :dso_prj_project
, dso_prj_coordinate = :dso_prj_coordinate
, dso_prj_degree = :dso_prj_degree
, dso_prj_zone = :dso_prj_zone
, dso_prj_source = :dso_prj_source
where dsoid = :dsoid
'''.format(dso_center_wgs84, dso_geo_bb_wgs84, dso_geo_wgs84),
params
)
return CResult.merge_result(self.Success, '空间元数据处理完毕!')
def process_mission(self, dataset):
storage_id = dataset.value_by_name(0, 'root_directory_id', '')
storage_root_path = dataset.value_by_name(0, 'root_directory', '')
CLogger().debug('storage_id: {0}'.format(storage_id))
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(
dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo, ds_retry_times))
self.update_status(storage_id, process_result,
self.ProcStatus_Error)
return process_result
sql_check_root_storage_dir_exist = '''
select dsdid
from dm2_storage_directory
where dsdid = :dsdid
'''
sql_update_root_storage_dir = '''
update dm2_storage_directory
set dsdParentID = '-1', dsdDirectory = '', dsdDirtype = {1}
, dsdDirectoryName = '', dsdPath = ''
, dsdDirCreateTime = :dsddircreatetime, dsdDirLastModifyTime = :dsddirlastmodifytime
, dsdLastModifyTime = Now(), dsd_directory_valid = {0}
where dsdid = :dsdid
'''.format(self.File_Status_Unknown, self.Dir_Type_Root)
sql_insert_root_storage_dir = '''
insert into dm2_storage_directory(
dsdid, dsdparentid, dsdstorageid, dsddirectory, dsddirtype, dsdlastmodifytime
, dsddirectoryname, dsd_directory_valid, dsdpath, dsddircreatetime, dsddirlastmodifytime)
values(:dsdid, '-1', :dsdStorageID, '', {1}, Now()
, '', {0}, '', :dsddircreatetime, :dsddirlastmodifytime
)
'''.format(self.File_Status_Unknown, self.Dir_Type_Root)
try:
db = CFactory().give_me_db(self.get_mission_db_id())
params = dict()
params['dsdid'] = storage_id
params['dsdStorageID'] = storage_id
if CFile.file_or_path_exist(storage_root_path):
params['dsdDirCreateTime'] = CFile.file_modify_time(
storage_root_path)
params['dsddirlastmodifytime'] = CFile.file_modify_time(
storage_root_path)
if db.if_exists(sql_check_root_storage_dir_exist, params):
db.execute(sql_update_root_storage_dir, params)
else:
db.execute(sql_insert_root_storage_dir, params)
process_result = CResult.merge_result(CResult.Success, '存储扫描处理成功')
self.update_status(storage_id, process_result)
return process_result
except DBException as err:
process_result = CResult.merge_result(
CResult.Exception, '存储扫描失败, 原因为{0}'.format(err.__str__))
self.update_status(storage_id, process_result)
return process_result
def open_source(self) -> CDataSetSeqReader:
option_source = CUtils.dict_value_by_name(self.__config, self.Name_Source, None)
if option_source is None:
raise Exception('数据源未设置, 请检查修正后重试! ')
type_source = CUtils.dict_value_by_name(option_source, self.Name_Type, self.Name_Table)
if CUtils.equal_ignore_case(type_source, self.Name_ShapeFile):
source_data_option = CUtils.dict_value_by_name(option_source, type_source, None)
source_data_filename = CUtils.dict_value_by_name(source_data_option, self.Name_Name, None)
if CUtils.equal_ignore_case(source_data_filename, ''):
raise Exception('[{0}]类型数据源未设置文件名, 请检查修正后重试! '.format(type_source))
if not CFile.file_or_path_exist(source_data_filename):
raise Exception('[{0}]类型数据源设置的文件[{1}]不存在, 请检查修正后重试! '.format(type_source, source_data_filename))
cpg_file_name = CFile.change_file_ext(source_data_filename, 'cpg')
encoding = CResource.Encoding_GBK
if CFile.file_or_path_exist(cpg_file_name):
encoding = CFile.file_2_str(cpg_file_name)
self.__source_data = CVectorDataSets(source_data_filename)
self.__source_data.open(encoding)
return self.__source_data.layer_by_index(0)
elif CUtils.equal_ignore_case(type_source, self.Name_Vector_DataSet):
source_data_option = CUtils.dict_value_by_name(option_source, type_source, None)
source_data_filename = CUtils.dict_value_by_name(source_data_option, self.Name_Name, None)
source_data_layer = CUtils.dict_value_by_name(source_data_option, self.Name_Layer, None)
if CUtils.equal_ignore_case(source_data_filename, ''):
raise Exception('[{0}]类型数据源未设置文件名, 请检查修正后重试! '.format(type_source))
if CUtils.equal_ignore_case(source_data_layer, ''):
raise Exception('[{0}]类型数据源未设置图层名, 请检查修正后重试! '.format(type_source))
if not CFile.file_or_path_exist(source_data_filename):
raise Exception('[{0}]类型数据源设置的文件[{1}]不存在, 请检查修正后重试! '.format(type_source, source_data_filename))
self.__source_data = CVectorDataSets(source_data_filename)
self.__source_data.open()
layer_data = self.__source_data.layer_by_name(source_data_layer)
if layer_data is None:
raise Exception('[{0}]类型数据源设置的图层[{1}]不存在, 请检查修正后重试! '.format(type_source, source_data_layer))
return layer_data
elif CUtils.equal_ignore_case(type_source, self.Name_Table):
source_data_option = CUtils.dict_value_by_name(option_source, type_source, None)
source_database = CUtils.dict_value_by_name(source_data_option, self.Name_DataBase,
self.DB_Server_ID_Default)
source_data_table_name = CUtils.dict_value_by_name(source_data_option, self.Name_Name, None)
if CUtils.equal_ignore_case(source_data_table_name, ''):
raise Exception('[{0}]类型数据源未设置数据表名, 请检查修正后重试! '.format(self.Name_Table))
data_set = CFactory().give_me_db(source_database).all_row(
'select * from {0}'.format(self.replace_placeholder(source_data_table_name)),
self.__params
)
self.__source_data = CDBDataSetSeqReader(data_set)
return self.__source_data
elif CUtils.equal_ignore_case(type_source, self.Name_Table):
source_data_option = CUtils.dict_value_by_name(option_source, type_source, None)
source_database = CUtils.dict_value_by_name(source_data_option, self.Name_DataBase,
self.DB_Server_ID_Default)
source_data_sql = CUtils.dict_value_by_name(source_data_option, self.Name_SQL, None)
if CUtils.equal_ignore_case(source_data_sql, ''):
raise Exception('[{0}]类型数据源未设置查询语句, 请检查修正后重试! '.format(type_source))
data_set = CFactory().give_me_db(source_database).all_row(
self.replace_placeholder(source_data_sql),
self.__params
)
self.__source_data = CDBDataSetSeqReader(data_set)
return self.__source_data
else:
raise Exception('系统不支持类型为[{0}]的数据源, 请检查修正后重试! '.format(type_source))
def process_mission(self, dataset) -> str:
"""
:param dataset:
:return:
"""
ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '')
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_storage_title = dataset.value_by_name(0, 'query_storage_title', '')
ds_storage_root_dir = dataset.value_by_name(0, 'query_rootpath', '')
ds_ib_directory_name = dataset.value_by_name(0,
'query_ib_relation_dir',
'')
ds_ib_directory_id = dataset.value_by_name(0,
'query_ib_relation_dir_id',
'')
ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '')
# 按需要再开启
# ds_ib_option = CUtils.any_2_str(dataset.value_by_name(0, 'query_ib_option', ''))
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format(
ds_storage_title,
CFile.join_file(ds_storage_root_dir, ds_ib_directory_name)))
else:
CLogger().debug('正在入库的是存储[{0}]下的目录[{1}]'.format(
ds_storage_title, ds_storage_root_dir))
try:
ds_ib_information_updated = False
# 检查目录名格式并自动修正
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
ds_ib_directory = CFile.unify(
CFile.add_prefix(ds_ib_directory_name))
if not CUtils.equal_ignore_case(ds_ib_directory,
ds_ib_directory_name):
ds_ib_directory_name = ds_ib_directory
ds_ib_information_updated = True
if CUtils.equal_ignore_case(ds_ib_batch_no, ''):
ds_ib_batch_no = CFactory().give_me_db(
self.get_mission_db_id()).seq_next_value(
self.Seq_Type_Date_AutoInc)
ds_ib_information_updated = True
if CUtils.equal_ignore_case(ds_ib_directory_id, ''):
ds_ib_directory_id = CUtils.one_id()
ds_ib_information_updated = True
if ds_ib_information_updated:
self.correct_ib_information(ds_ib_id, ds_ib_directory_name,
ds_ib_batch_no, ds_ib_directory_id)
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
ib_full_directory = CFile.join_file(ds_storage_root_dir,
ds_ib_directory_name)
else:
ib_full_directory = ds_storage_root_dir
self.clear_anything_in_directory(ds_ib_id)
metadata_rule_file_name = CFile.join_file(
ib_full_directory, self.FileName_MetaData_Rule)
metadata_rule_content = ''
if CFile.file_or_path_exist(metadata_rule_file_name):
try:
metadata_rule_content = CXml.file_2_str(
metadata_rule_file_name)
CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format(
ib_full_directory, metadata_rule_content))
except Exception as error:
result = CResult.merge_result(
self.Failure,
'在目录[{0}]下发现元数据规则文件, 但它的格式不合法, 详细错误为: [{1}]'.format(
ib_full_directory, error.__str__()))
self.update_inbound_qi_result(ds_ib_id, result)
return result
path_obj = CDMPathInfo(self.FileType_Dir, ib_full_directory,
ds_storage_id,
ds_ib_directory_id, ds_storage_id, None,
self.get_mission_db_id(),
metadata_rule_content)
if path_obj.white_black_valid():
path_obj.db_check_and_update(ds_ib_id)
result = CResult.merge_result(
self.Success, '目录[{0}]的入库质检任务创建成功, 系统正在质检, 请稍后...'.format(
ib_full_directory))
else:
result = CResult.merge_result(
self.Failure,
'目录[{0}]未通过黑白名单检验, 不允许入库! '.format(ib_full_directory))
self.update_inbound_qi_result(ds_ib_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure, '目录[{0}]的入库质检任务创建过程出现错误, 详细错误为: [{1}]'.format(
CFile.join_file(ds_storage_root_dir, ds_ib_directory_name),
error.__str__()))
self.update_inbound_qi_result(ds_ib_id, result)
return result
def db_check_and_update_metadata_rule(self, metadata_rule_file_name):
"""
检查并判断指定的元数据扫描规则文件是否与数据库中的记录相等
1. 如果和记录中的不同
删除当前目录下的所有子目录, 文件 和对象
更新记录中的规则
设置子目录扫描状态为正常
2. 如果和记录中的相同
返回
:param metadata_rule_file_name:
:return:
"""
metadata_rule_content = ''
if CFile.file_or_path_exist(metadata_rule_file_name):
try:
metadata_rule_content = CXml.file_2_str(
metadata_rule_file_name)
CLogger().debug('在目录[{0}]下发现元数据规则文件, 它的内容为[{1}]'.format(
self.file_name_with_full_path, metadata_rule_content))
except:
pass
db_metadata_rule_content = self.ds_file_or_path.value_by_name(
0, 'dsdscanrule', '')
CLogger().debug('目录[{0}]在库中登记的规则内容为[{1}]'.format(
self.file_name_with_full_path, db_metadata_rule_content))
if CUtils.equal_ignore_case(metadata_rule_content,
db_metadata_rule_content):
CLogger().debug('目录[{0}]的规则内容与库中的相同'.format(
self.file_name_with_full_path))
return
else:
CLogger().debug('目录[{0}]的规则内容与库中的不同, 将清理目录下的文件, 重新入库!'.format(
self.file_name_with_full_path))
sql_update_path_scan_rule = '''
update dm2_storage_directory
set dsd_directory_valid = -1, dsdscanrule = :dsdScanRule
where dsdid = :dsdID
'''
sql_clear_files_of_path = '''
delete from dm2_storage_file
where dsfdirectoryid in (
select dsdid
from dm2_storage_directory
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1
)
'''
sql_clear_subpath_of_path = '''
delete from dm2_storage_directory
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory) = 1
and dsdid <> :dsdID
'''
engine = CFactory().give_me_db(self.db_server_id)
session = engine.give_me_session()
try:
params = dict()
params['dsdID'] = self.my_id
if metadata_rule_content == '':
params['dsdScanRule'] = None
else:
params['dsdScanRule'] = metadata_rule_content
engine.session_execute(session, sql_update_path_scan_rule, params)
params = dict()
params['dsdID'] = self.my_id
params['dsdStorageID'] = self.storage_id
params['dsdSubDirectory'] = CFile.join_file(
self.file_name_with_rel_path, '')
engine.session_execute(session, sql_clear_files_of_path, params)
engine.session_execute(session, sql_clear_subpath_of_path, params)
engine.session_commit(session)
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
engine.session_rollback(session)
finally:
engine.session_close(session)
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
完成 负责人 王学谦
:param parser:
:return:
"""
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
check_file_metadata_bus_exist = False
ext = self.Transformer_XML
metadata_name_with_path = CFile.join_file(file_path, '{0}M.xml'.format(file_main_name[:-1]))
if CFile.file_or_path_exist(metadata_name_with_path):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = metadata_name_with_path
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Error,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本文件缺少业务元数据'
}
)
else:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: self.metadata_bus_src_filename_with_path,
self.Name_ID: 'metadata_file',
self.Name_Title: '元数据文件',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '业务元数据[{0}]存在'.format(
CFile.file_name(self.metadata_bus_src_filename_with_path)
)
}
)
letter_location_FenFu = file_path.find('FenFu')
file_path_sub = file_path[letter_location_FenFu + 6:]
letter_location = file_path_sub.find(CFile.sep())
shp_path = CFile.join_file(file_path[:letter_location_FenFu + 5], file_path_sub[:letter_location])
shp_list = CFile.file_or_subpath_of_path(shp_path, '(?i).shp$', CFile.MatchType_Regex)
if len(shp_list) == 0:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: '',
self.Name_ID: 'shp_file',
self.Name_Title: '影像时相接边图',
self.Name_Result: self.QA_Result_Warn,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '本文件缺少影像时相接边图'
}
)
else:
parser.metadata.quality.append_total_quality(
{
self.Name_FileName: shp_list[0],
self.Name_ID: 'shp_file',
self.Name_Title: '影像时相接边图',
self.Name_Result: self.QA_Result_Pass,
self.Name_Group: self.QA_Group_Data_Integrity,
self.Name_Message: '影像时相接边图[{0}]存在'.format(shp_list[0])
}
)
if layer_name.upper().endswith('_SHAPE_INDEX'):
return False
elif CUtils.equal_ignore_case(layer_name, 'Selections'):
return False
elif CUtils.equal_ignore_case(layer_name, 'SelectedObjects'):
return False
return True
if __name__ == '__main__':
file_name = '/Users/wangxiya/Documents/我的测试数据/31.混合存储/测试数据/通用数据/矢量数据集/生态治理和水土保持监测数据库_黑岱沟露天煤矿_10017699_2020d1_2020-01-01.mdb'
# file_name = '/Users/wangxiya/Documents/我的测试数据/31.混合存储/测试数据/通用数据/矢量/矢量数据/正确数据/生态治理和水土保持监测数据库_利民煤矿_10017574_2020d1_2020-08-05/取排口.shp'
cpg_file_name = CFile.change_file_ext(file_name, 'cpg')
encoding = CResource.Encoding_GBK
if CFile.file_or_path_exist(cpg_file_name):
encoding = CFile.file_2_str(cpg_file_name)
vector_datasets = CVectorDataSets(file_name)
vector_datasets.open(encoding)
for vector_layer_index in range(vector_datasets.size()):
vector_dataset = vector_datasets.layer_by_index(vector_layer_index)
print('*' * 30)
print('{0}-{1}'.format(vector_dataset.dataset_name,
vector_dataset.size()))
print('*' * 10)
for field_index in range(vector_dataset.field_count()):
print('{0}.{1}'.format(vector_dataset.dataset_name,
vector_dataset.field_name(field_index)))
vector_dataset = vector_datasets.layer_by_index(0)
print('*' * 30)
def qa_file_custom(self, parser: CMetaDataParser):
"""
自定义的文件存在性质检, 发生在元数据解析之前
:param parser:
:return:
"""
super().qa_file_custom(parser)
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
file_name_before_six = file_main_name[0:6] # 截取前六位行政区划代码
# 查询行政区划代码对应的名称并拼接
try:
db = CFactory().give_me_db(self.file_info.db_server_id)
file_name_before_six_name = db.one_row("select gdstitle from ro_global_dim_space "
"where gdsid = '{0}'".format(file_name_before_six)) \
.value_by_name(0, 'gdstitle', None)
except:
file_name_before_six_name = ''
file_metadata_name = '{0}{1}'.format(file_name_before_six,
file_name_before_six_name)
check_file_metadata_bus_exist = False
ext = self.Transformer_DOM_MDB # 后缀名
metadata_main_name_with_path = CFile.join_file(file_path,
file_metadata_name)
# 拼接好元数据文件名称，并检查其是否存在
temp_metadata_bus_file = '{0}.{1}'.format(metadata_main_name_with_path,
ext)
if CFile.file_or_path_exist(temp_metadata_bus_file):
check_file_metadata_bus_exist = True
self.metadata_bus_transformer_type = ext
self.metadata_bus_src_filename_with_path = temp_metadata_bus_file
if not check_file_metadata_bus_exist:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
'',
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Error,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'本文件缺少业务元数据'
})
else:
parser.metadata.quality.append_total_quality({
self.Name_FileName:
self.metadata_bus_src_filename_with_path,
self.Name_ID:
'metadata_file',
self.Name_Title:
'元数据文件',
self.Name_Result:
self.QA_Result_Pass,
self.Name_Group:
self.QA_Group_Data_Integrity,
self.Name_Message:
'业务元数据[{0}]存在'.format(self.metadata_bus_src_filename_with_path)
})
def create_virtual_content(self) -> bool:
CFile.check_and_create_directory_itself(self.__work_root_dir__)
return CFile.file_or_path_exist(self.__work_root_dir__)
def create_virtual_content(self) -> bool:
return super().create_virtual_content() and CFile.file_or_path_exist(
self.__virtual_content_root_dir__)
def __find_module_obj(self, obj_id):
sql_query = '''
SELECT
dm2_storage_file.dsfid as query_file_id,
dm2_storage_file.dsfdirectoryid as query_directory_id,
dm2_storage_directory.dsddirlastmodifytime as query_directory_lastmodifytime,
dm2_storage_directory.dsdid as query_dataset_directory_id,
dm2_storage_object.*
FROM
dm2_storage_object
LEFT JOIN dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid
LEFT JOIN dm2_storage_directory on dm2_storage_directory.dsd_object_id = dm2_storage_object.dsoid
WHERE
dm2_storage_object.dsoid = '{0}'
'''.format(obj_id)
db_id = self._db_id # 数据库连接标识（查询的dm2的数管库）
dataset = CFactory().give_me_db(db_id).one_row(sql_query)
object_id = dataset.value_by_name(0, 'dsoid', '')
object_name = dataset.value_by_name(0, 'dsoobjectname', '')
quality_info = dataset.value_by_name(0, 'dso_quality', '')
quality_info_xml = CXml()
quality_info_xml.load_xml(quality_info) # 加载查询出来的xml
db_id_distribution = self.DB_Server_ID_Distribution # 同步处理的目标数据库标识id
# 构建数据对象object对应的识别插件，获取get_information里面的Plugins_Info_Module_Distribute_Engine信息
class_classified_obj = CObject.get_plugins_instance_by_object_id(
db_id, object_id)
if class_classified_obj is not None:
plugins_info = class_classified_obj.get_information()
obj_type_code = CUtils.dict_value_by_name(
plugins_info, class_classified_obj.Plugins_Info_Type_Code, '')
distribution_file_main_name = CUtils.dict_value_by_name(
plugins_info,
class_classified_obj.Plugins_Info_Module_Distribute_Engine,
'') # 对应同步文件的类名称
# 判断同步插件文件是否存在
access_modules_root_dir = CSys.get_metadata_data_access_modules_root_dir(
)
distribution_root_dir = CFile.join_file(access_modules_root_dir,
self.Name_Distribution)
distribution_file = CFile.join_file(
distribution_root_dir,
'{0}.py'.format(distribution_file_main_name))
if CFile.file_or_path_exist(distribution_file):
# 构建同步对象
distribution_obj = CObject.create_module_distribution_instance(
'{0}.{1}'.format(
CSys.get_metadata_data_access_modules_root_name(),
self.Name_Distribution), distribution_file_main_name,
db_id_distribution, object_id, object_name, obj_type_code,
quality_info_xml, dataset)
distribution_obj.set_class_plugins(class_classified_obj)
result = CResult.merge_result(self.Success, '数据的信息提取完成')
return distribution_obj, result
else:
result = CResult.merge_result(self.Failure,
'同步失败，因为本类数据的同步模块未建立')
return None, result
else:
result = CResult.merge_result(self.Failure, '同步失败，因为本类数据的识别模块缺失')
return None, result
def virtual_content_valid(self) -> bool:
return CFile.file_or_path_exist(self.__virtual_content_root_dir__)