def __condition_list_2_sql(self, field, value_list, quoted_value): if quoted_value: in_sql = CUtils.list_2_str(value_list, "'", ", ", "'", True) else: in_sql = CUtils.list_2_str(value_list, "", ", ", "", True) if CUtils.equal_ignore_case(in_sql, ''): return "" else: return "{0} in ({1})".format(field, in_sql)
def set_array(self, src_array: list): """ 设置value为数组 :param src_array: :return: """ if len(src_array) == 0: self.reset() else: if isinstance(src_array[0], str): array_text = CUtils.list_2_str(src_array, "'", ',', "'") else: array_text = CUtils.list_2_str(src_array, '', ',', '') self._value = { CResource.Name_Text: array_text, CResource.Name_Type: CResource.DataValueType_Value, CResource.Name_Value: src_array }
def execute(self) -> str: inbound_ib_n_list = CFactory().give_me_db( self.get_mission_db_id()).all_row(''' select dsiid as query_ib_id , dsiotheroption as query_ib_option , dsidirectory as query_ib_relation_dir , dsidirectoryid as query_ib_dir_id , dsitargetstorageid as query_ib_target_storage_id from dm2_storage_inbound where dsi_na_status = {0} '''.format(self.ProcStatus_WaitConfirm)) if inbound_ib_n_list.is_empty(): return CResult.merge_result(CResult.Success, '本次没有需要检查的通知任务!') for data_index in range(inbound_ib_n_list.size()): ds_ib_id = inbound_ib_n_list.value_by_name(data_index, 'query_ib_id', '') ds_ib_option = CUtils.any_2_str( inbound_ib_n_list.value_by_name(data_index, 'query_ib_option', '')) ds_ib_directory_name = inbound_ib_n_list.value_by_name( data_index, 'query_ib_relation_dir', '') ds_ib_directory_id = inbound_ib_n_list.value_by_name( data_index, 'query_ib_dir_id', '') ds_ib_target_storage_id = inbound_ib_n_list.value_by_name( data_index, 'query_ib_target_storage_id', '') module_name_list = CJson.json_attr_value( ds_ib_option, self.Path_IB_Opt_Notify_module, None) if module_name_list is None: modules_root_dir = CSys.get_metadata_data_access_modules_root_dir( ) module_file_list = CFile.file_or_subpath_of_path( modules_root_dir, '{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py)) module_name_list = list() for module_file in module_file_list: module_name_list.append(CFile.file_main_name(module_file)) CLogger().debug('正在检查入库批次[ds_ib_id]的通知进度...'.format(ds_ib_id)) try: # 所有通知对象的统计数 sql_record_total_count = CUtils.replace_placeholder( ''' select count(*) from dm2_storage_obj_na where dson_app_id in ($module_name_list) and dson_object_id in ( select dsoid from dm2_storage_object where dso_ib_id = :ib_id ) ''', { 'module_name_list': CUtils.list_2_str(module_name_list, "'", ',', "'") }) record_total_count = CFactory().give_me_db( self.get_mission_db_id()).one_value( sql_record_total_count, {'ib_id': ds_ib_id}, 0) if record_total_count == 0: self.update_inbound_na_result( ds_ib_id, CResult.merge_result( self.Failure, '入库任务下没有可通知的数据, 请检查异常情况! '.format(ds_ib_id))) continue # 已经完成的通知对象的统计数, 包括正常完成和错误的 sql_record_finished_count = CUtils.replace_placeholder( ''' select count(*) from dm2_storage_obj_na where dson_notify_status in ({0}, {1}) and dson_app_id in ($module_name_list) and dson_object_id in ( select dsoid from dm2_storage_object where dso_ib_id = :ib_id ) '''.format(self.ProcStatus_Finished, self.ProcStatus_Error), { 'module_name_list': CUtils.list_2_str(module_name_list, "'", ',', "'") }) record_finished_count = CFactory().give_me_db( self.get_mission_db_id()).one_value( sql_record_finished_count, {'ib_id': ds_ib_id}, 0) # 错误的记录数 sql_record_error_count = CUtils.replace_placeholder( ''' select count(*) from dm2_storage_obj_na where dson_notify_status = {0} and dson_app_id in ($module_name_list) and dson_object_id in ( select dsoid from dm2_storage_object where dso_ib_id = :ib_id ) '''.format(self.ProcStatus_Error), { 'module_name_list': CUtils.list_2_str(module_name_list, "'", ',', "'") }) record_error_count = CFactory().give_me_db( self.get_mission_db_id()).one_value( sql_record_error_count, {'ib_id': ds_ib_id}, 0) if record_total_count != record_finished_count: message = '入库任务[{0}]下的数据正在通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个...'.format( ds_ib_id, record_total_count, record_finished_count, record_error_count) CLogger().debug(message) self.update_inbound_na_progress( ds_ib_id, CResult.merge_result(self.Failure, message)) else: message = '入库任务[{0}]下的数据已经通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个, 请检查修正! '.format( ds_ib_id, record_total_count, record_finished_count, record_error_count) CLogger().debug(message) self.update_inbound_na_result( ds_ib_id, CResult.merge_result(self.Success, message)) except Exception as error: self.update_inbound_na_result( ds_ib_id, CResult.merge_result( self.Failure, '入库任务下的数据通知其他子系统过程中出现异常情况, 详细错误信息为: [{1}]'.format( ds_ib_id, error.__str__()))) continue return CResult.merge_result(self.Success, '本次通知监控任务成功结束!')
def search(self, module_name: str, search_json_obj: CJson, other_option: dict = None) -> CDataSet: """ 根据搜索条件json, 检索符合要求的对象, 并以数据集的方式返回如下字段: 1. object_id 1. object_name 1. object_type 1. object_data_type 1. object_parent_id 1. object_size 1. object_lastmodifytime :param module_name: 模块名称 :param search_json_obj: :param other_option: :return: """ if search_json_obj is None: return CDataSet() params_search = dict() sql_from = '' sql_where = '' if (not CUtils.equal_ignore_case(module_name, self.ModuleName_MetaData)) and \ (not CUtils.equal_ignore_case(module_name, '')): # sql_where = "dm2_storage_object.dso_da_result#>>'{{{0},result}}'='pass'".format(module_name) sql_from = ', dm2_storage_obj_na ' sql_where = " dm2_storage_obj_na.dson_app_id = 'module_name' " condition_obj_access = search_json_obj.xpath_one(self.Name_Access, self.DataAccess_Pass) if not CUtils.equal_ignore_case(condition_obj_access, ''): condition = "dm2_storage_obj_na.dson_object_access = '{0}'".format(CUtils.any_2_str(condition_obj_access)) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_inbound_id = search_json_obj.xpath_one(self.Name_InBound, None) if not CUtils.equal_ignore_case(condition_inbound_id, ''): condition = "dm2_storage_obj.dso_ib_id = '{0}'".format(CUtils.any_2_str(condition_inbound_id)) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_tag = search_json_obj.xpath_one(self.Name_Tag, None) if condition_tag is not None: if isinstance(condition_tag, list): condition = CUtils.list_2_str(condition_tag, "'", ", ", "'", True) else: condition = CUtils.list_2_str([condition_tag], "'", ", ", "'", True) if not CUtils.equal_ignore_case(condition, ''): condition = 'dm2_storage_object.dsotags @ > array[{0}]:: CHARACTER VARYING[]'.format(condition) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_id = search_json_obj.xpath_one(self.Name_ID, None) if condition_id is not None: if isinstance(condition_id, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodid', condition_id, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodid', condition_id, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_name = search_json_obj.xpath_one(self.Name_Name, None) if condition_name is not None: if isinstance(condition_name, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodname', condition_name, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodname', condition_name, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_type = search_json_obj.xpath_one(self.Name_Type, None) if condition_type is not None: if isinstance(condition_type, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodtype', condition_type, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodtype', condition_type, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') condition_group = search_json_obj.xpath_one(self.Name_Group, None) if condition_group is not None: if isinstance(condition_group, list): condition = self.__condition_list_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True) else: condition = self.__condition_value_like_2_sql('dm2_storage_object_def.dsodgroup', condition_group, True) sql_where = CUtils.str_append(sql_where, condition, ' and ') if not CUtils.equal_ignore_case(sql_where, ''): sql_where = ' and {0}'.format(sql_where) sql_search = ''' select dm2_storage_object.dsoid as object_id , dm2_storage_object.dsoobjectname as object_name , dm2_storage_object.dsoobjecttype as object_type , dm2_storage_object.dsodatatype as object_data_type , dm2_storage_object.dsoparentobjid as object_parent_id , dm2_storage_object.dso_volumn_now as object_size , dm2_storage_object.dso_obj_lastmodifytime as object_lastmodifytime from dm2_storage_object, dm2_storage_object_def {0} where dm2_storage_object.dsoobjecttype = dm2_storage_object_def.dsodid and dm2_storage_object.dsoid = dm2_storage_obj_na.dson_object_id {1} '''.format(sql_from, sql_where) return CFactory().give_me_db(self.db_server_id).all_row(sql_search)
def parser_metadata_custom(self, parser: CMetaDataParser) -> str: """ 自定义的元数据解析, 在所有质检和其他处理之后触发 :param parser: :return: """ meta_data_json = parser.metadata.metadata_json() if meta_data_json is None: return CResult.merge_result( self.Success, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 无法获取JSON格式的元数据! '.format( self.file_info.file_name_with_full_path, ) ) json_data_source = meta_data_json.xpath_one('datasource', None) layer_list = meta_data_json.xpath_one(self.Name_Layers, None) if layer_list is None: return CResult.merge_result( self.Success, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 元数据中无法找到layers节点! '.format( self.file_info.file_name_with_full_path, ) ) mdb_ib_id = CFactory().give_me_db(self.file_info.db_server_id).one_value( ''' select dso_ib_id from dm2_storage_object where dsoid = :object_id ''', { 'object_id': parser.object_id } ) error_message_list = [] table = CTable() table.load_info(self.file_info.db_server_id, self.TableName_DM_Object) for layer in layer_list: layer_name = CUtils.dict_value_by_name(layer, self.Name_Name, '') if CUtils.equal_ignore_case(layer_name, ''): continue layer_alias_name = CUtils.dict_value_by_name(layer, self.Name_Description, layer_name) layer_metadata_json = CJson() layer_metadata_json.set_value_of_name('datasource', json_data_source) layer_metadata_json.set_value_of_name('layer_count', 1) layer_metadata_json.set_value_of_name('layers', [layer]) layer_metadata_text = layer_metadata_json.to_json() try: sql_find_layer_existed = ''' select dsoid as layer_id_existed from dm2_storage_object where upper(dsoobjectname) = upper(:layer_name) and dsoparentobjid = :object_id ''' layer_id_existed = CFactory().give_me_db(self.file_info.db_server_id).one_value( sql_find_layer_existed, { 'layer_name': layer_name, 'object_id': parser.object_id } ) if layer_id_existed is None: layer_id_existed = CUtils.one_id() table.column_list.reset() table.column_list.column_by_name('dsoid').set_value(layer_id_existed) table.column_list.column_by_name('dsoobjectname').set_value(layer_name) table.column_list.column_by_name('dsoobjecttype').set_value( CUtils.dict_value_by_name( self.get_information(), self.Plugins_Info_Child_Layer_Plugins_Name, '' ) ) table.column_list.column_by_name('dsodatatype').set_value( CUtils.dict_value_by_name( self.get_information(), self.Plugins_Info_Child_Layer_Data_Type, '' ) ) table.column_list.column_by_name('dsoalphacode').set_value(CUtils.alpha_text(layer_name)) table.column_list.column_by_name('dsoaliasname').set_value(layer_alias_name) table.column_list.column_by_name('dsoparentobjid').set_value(parser.object_id) table.column_list.column_by_name('dso_ib_id').set_value(mdb_ib_id) table.column_list.column_by_name('dsometadatatext').set_value(layer_metadata_text) table.column_list.column_by_name('dsometadatajson').set_value(layer_metadata_text) table.column_list.column_by_name('dsometadataparsestatus').set_value(self.ProcStatus_InQueue) table.column_list.column_by_name('dsotagsparsestatus').set_value(self.ProcStatus_InQueue) table.column_list.column_by_name('dsodetailparsestatus').set_value(self.ProcStatus_InQueue) result = table.save_data() if not CResult.result_success(result): error_message_list.append( '图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format( layer_name, CResult.result_message(result) ) ) except Exception as error: error_message_list.append('图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(layer_name, error.__str__())) if len(error_message_list) > 0: return CResult.merge_result( self.Failure, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 详细情况如下: \n{1}'.format( self.file_info.file_name_with_full_path, CUtils.list_2_str(error_message_list, '', '\n', '', True) ) ) else: return CResult.merge_result( self.Success, '数据[{0}]的自定义元数据解析完毕! '.format( self.file_info.file_name_with_full_path, ) )