def __condition_list_2_sql(self, field, value_list, quoted_value):
if quoted_value:
in_sql = CUtils.list_2_str(value_list, "'", ", ", "'", True)
else:
in_sql = CUtils.list_2_str(value_list, "", ", ", "", True)
if CUtils.equal_ignore_case(in_sql, ''):
return ""
else:
return "{0} in ({1})".format(field, in_sql)
def set_array(self, src_array: list):
"""
设置value为数组
:param src_array:
:return:
"""
if len(src_array) == 0:
self.reset()
else:
if isinstance(src_array[0], str):
array_text = CUtils.list_2_str(src_array, "'", ',', "'")
else:
array_text = CUtils.list_2_str(src_array, '', ',', '')
self._value = {
CResource.Name_Text: array_text,
CResource.Name_Type: CResource.DataValueType_Value,
CResource.Name_Value: src_array
}
def execute(self) -> str:
inbound_ib_n_list = CFactory().give_me_db(
self.get_mission_db_id()).all_row('''
select
dsiid as query_ib_id
, dsiotheroption as query_ib_option
, dsidirectory as query_ib_relation_dir
, dsidirectoryid as query_ib_dir_id
, dsitargetstorageid as query_ib_target_storage_id
from dm2_storage_inbound
where dsi_na_status = {0}
'''.format(self.ProcStatus_WaitConfirm))
if inbound_ib_n_list.is_empty():
return CResult.merge_result(CResult.Success, '本次没有需要检查的通知任务！')
for data_index in range(inbound_ib_n_list.size()):
ds_ib_id = inbound_ib_n_list.value_by_name(data_index,
'query_ib_id', '')
ds_ib_option = CUtils.any_2_str(
inbound_ib_n_list.value_by_name(data_index, 'query_ib_option',
''))
ds_ib_directory_name = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_relation_dir', '')
ds_ib_directory_id = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_dir_id', '')
ds_ib_target_storage_id = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_target_storage_id', '')
module_name_list = CJson.json_attr_value(
ds_ib_option, self.Path_IB_Opt_Notify_module, None)
if module_name_list is None:
modules_root_dir = CSys.get_metadata_data_access_modules_root_dir(
)
module_file_list = CFile.file_or_subpath_of_path(
modules_root_dir,
'{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py))
module_name_list = list()
for module_file in module_file_list:
module_name_list.append(CFile.file_main_name(module_file))
CLogger().debug('正在检查入库批次[ds_ib_id]的通知进度...'.format(ds_ib_id))
try:
# 所有通知对象的统计数
sql_record_total_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
''', {
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_total_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_total_count, {'ib_id': ds_ib_id}, 0)
if record_total_count == 0:
self.update_inbound_na_result(
ds_ib_id,
CResult.merge_result(
self.Failure,
'入库任务下没有可通知的数据, 请检查异常情况! '.format(ds_ib_id)))
continue
# 已经完成的通知对象的统计数, 包括正常完成和错误的
sql_record_finished_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_notify_status in ({0}, {1})
and dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
'''.format(self.ProcStatus_Finished,
self.ProcStatus_Error),
{
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_finished_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_finished_count, {'ib_id': ds_ib_id}, 0)
# 错误的记录数
sql_record_error_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_notify_status = {0}
and dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
'''.format(self.ProcStatus_Error), {
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_error_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_error_count, {'ib_id': ds_ib_id}, 0)
if record_total_count != record_finished_count:
message = '入库任务[{0}]下的数据正在通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个...'.format(
ds_ib_id, record_total_count, record_finished_count,
record_error_count)
CLogger().debug(message)
self.update_inbound_na_progress(
ds_ib_id, CResult.merge_result(self.Failure, message))
else:
message = '入库任务[{0}]下的数据已经通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个, 请检查修正! '.format(
ds_ib_id, record_total_count, record_finished_count,
record_error_count)
CLogger().debug(message)
self.update_inbound_na_result(
ds_ib_id, CResult.merge_result(self.Success, message))
except Exception as error:
self.update_inbound_na_result(
ds_ib_id,
CResult.merge_result(
self.Failure,
'入库任务下的数据通知其他子系统过程中出现异常情况, 详细错误信息为: [{1}]'.format(
ds_ib_id, error.__str__())))
continue
return CResult.merge_result(self.Success, '本次通知监控任务成功结束！')
def search(self, module_name: str, search_json_obj: CJson, other_option: dict = None) -> CDataSet:
"""
根据搜索条件json, 检索符合要求的对象, 并以数据集的方式返回如下字段:
1. object_id
1. object_name
1. object_type
1. object_data_type
1. object_parent_id
1. object_size
1. object_lastmodifytime
:param module_name: 模块名称
:param search_json_obj:
:param other_option:
:return:
"""
if search_json_obj is None:
return CDataSet()
params_search = dict()
sql_from = ''
sql_where = ''
if (not CUtils.equal_ignore_case(module_name, self.ModuleName_MetaData)) and \
(not CUtils.equal_ignore_case(module_name, '')):
# sql_where = "dm2_storage_object.dso_da_result#>>'{{{0},result}}'='pass'".format(module_name)
sql_from = ', dm2_storage_obj_na '
sql_where = " dm2_storage_obj_na.dson_app_id = 'module_name' "
condition_obj_access = search_json_obj.xpath_one(self.Name_Access, self.DataAccess_Pass)
if not CUtils.equal_ignore_case(condition_obj_access, ''):
condition = "dm2_storage_obj_na.dson_object_access = '{0}'".format(CUtils.any_2_str(condition_obj_access))
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_inbound_id = search_json_obj.xpath_one(self.Name_InBound, None)
if not CUtils.equal_ignore_case(condition_inbound_id, ''):
condition = "dm2_storage_obj.dso_ib_id = '{0}'".format(CUtils.any_2_str(condition_inbound_id))
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_tag = search_json_obj.xpath_one(self.Name_Tag, None)
if condition_tag is not None:
if isinstance(condition_tag, list):
condition = CUtils.list_2_str(condition_tag, "'", ", ", "'", True)
else:
condition = CUtils.list_2_str([condition_tag], "'", ", ", "'", True)
if not CUtils.equal_ignore_case(condition, ''):
condition = 'dm2_storage_object.dsotags @ > array[{0}]:: CHARACTER VARYING[]'.format(condition)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_id = search_json_obj.xpath_one(self.Name_ID, None)
if condition_id is not None:
if isinstance(condition_id, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodid', condition_id, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodid', condition_id, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_name = search_json_obj.xpath_one(self.Name_Name, None)
if condition_name is not None:
if isinstance(condition_name, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodname', condition_name, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodname', condition_name, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_type = search_json_obj.xpath_one(self.Name_Type, None)
if condition_type is not None:
if isinstance(condition_type, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodtype', condition_type, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodtype', condition_type, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
condition_group = search_json_obj.xpath_one(self.Name_Group, None)
if condition_group is not None:
if isinstance(condition_group, list):
condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True)
else:
condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True)
sql_where = CUtils.str_append(sql_where, condition, ' and ')
if not CUtils.equal_ignore_case(sql_where, ''):
sql_where = ' and {0}'.format(sql_where)
sql_search = '''
select dm2_storage_object.dsoid as object_id
, dm2_storage_object.dsoobjectname as object_name
, dm2_storage_object.dsoobjecttype as object_type
, dm2_storage_object.dsodatatype as object_data_type
, dm2_storage_object.dsoparentobjid as object_parent_id
, dm2_storage_object.dso_volumn_now as object_size
, dm2_storage_object.dso_obj_lastmodifytime as object_lastmodifytime
from dm2_storage_object, dm2_storage_object_def {0}
where dm2_storage_object.dsoobjecttype = dm2_storage_object_def.dsodid
and dm2_storage_object.dsoid = dm2_storage_obj_na.dson_object_id
{1}
'''.format(sql_from, sql_where)
return CFactory().give_me_db(self.db_server_id).all_row(sql_search)
def parser_metadata_custom(self, parser: CMetaDataParser) -> str:
"""
自定义的元数据解析, 在所有质检和其他处理之后触发
:param parser:
:return:
"""
meta_data_json = parser.metadata.metadata_json()
if meta_data_json is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 无法获取JSON格式的元数据! '.format(
self.file_info.file_name_with_full_path,
)
)
json_data_source = meta_data_json.xpath_one('datasource', None)
layer_list = meta_data_json.xpath_one(self.Name_Layers, None)
if layer_list is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 元数据中无法找到layers节点! '.format(
self.file_info.file_name_with_full_path,
)
)
mdb_ib_id = CFactory().give_me_db(self.file_info.db_server_id).one_value(
'''
select dso_ib_id
from dm2_storage_object
where dsoid = :object_id
''',
{
'object_id': parser.object_id
}
)
error_message_list = []
table = CTable()
table.load_info(self.file_info.db_server_id, self.TableName_DM_Object)
for layer in layer_list:
layer_name = CUtils.dict_value_by_name(layer, self.Name_Name, '')
if CUtils.equal_ignore_case(layer_name, ''):
continue
layer_alias_name = CUtils.dict_value_by_name(layer, self.Name_Description, layer_name)
layer_metadata_json = CJson()
layer_metadata_json.set_value_of_name('datasource', json_data_source)
layer_metadata_json.set_value_of_name('layer_count', 1)
layer_metadata_json.set_value_of_name('layers', [layer])
layer_metadata_text = layer_metadata_json.to_json()
try:
sql_find_layer_existed = '''
select dsoid as layer_id_existed
from dm2_storage_object
where upper(dsoobjectname) = upper(:layer_name)
and dsoparentobjid = :object_id
'''
layer_id_existed = CFactory().give_me_db(self.file_info.db_server_id).one_value(
sql_find_layer_existed,
{
'layer_name': layer_name,
'object_id': parser.object_id
}
)
if layer_id_existed is None:
layer_id_existed = CUtils.one_id()
table.column_list.reset()
table.column_list.column_by_name('dsoid').set_value(layer_id_existed)
table.column_list.column_by_name('dsoobjectname').set_value(layer_name)
table.column_list.column_by_name('dsoobjecttype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Plugins_Name,
''
)
)
table.column_list.column_by_name('dsodatatype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Data_Type,
''
)
)
table.column_list.column_by_name('dsoalphacode').set_value(CUtils.alpha_text(layer_name))
table.column_list.column_by_name('dsoaliasname').set_value(layer_alias_name)
table.column_list.column_by_name('dsoparentobjid').set_value(parser.object_id)
table.column_list.column_by_name('dso_ib_id').set_value(mdb_ib_id)
table.column_list.column_by_name('dsometadatatext').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadatajson').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadataparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsotagsparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsodetailparsestatus').set_value(self.ProcStatus_InQueue)
result = table.save_data()
if not CResult.result_success(result):
error_message_list.append(
'图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(
layer_name,
CResult.result_message(result)
)
)
except Exception as error:
error_message_list.append('图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(layer_name, error.__str__()))
if len(error_message_list) > 0:
return CResult.merge_result(
self.Failure,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 详细情况如下: \n{1}'.format(
self.file_info.file_name_with_full_path,
CUtils.list_2_str(error_message_list, '', '\n', '', True)
)
)
else:
return CResult.merge_result(
self.Success,
'数据[{0}]的自定义元数据解析完毕! '.format(
self.file_info.file_name_with_full_path,
)
)
