def __a_check_file__(cls, result_template: dict, file_name_with_path: str, qa_items: dict) -> list: """ 根据规则, 验证文件的合法性 完成 负责人 赵宇飞 在这里对文件的其他内容进行质检, 目前实现了检查文件大小, 请参考__a_check_file_size__完善其他内容, 包括并不限于 验证文件可读性, 验证元数据文件可读性, 验证元数据文件的格式(xml\json), 以便于后面在处理元数据时, 不会出现异常 :param result_template 检查结果的模板 :param file_name_with_path: 文件名 :param qa_items: 检查项目, keywords :return: """ result_list = list() result_list.append( cls.__a_check_file_size__( result_template, file_name_with_path, CJson.dict_attr_by_path( qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Min), -1), CJson.dict_attr_by_path( qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Max), -1))) result_list.append( cls.__a_check_file_format__( result_template, file_name_with_path, CJson.dict_attr_by_path(qa_items, cls.Name_Format, None))) return result_list
def get_dest_storage(self, ib_batch_no, need_storage_size, ib_option, ib_schema, dataset_xml): """ 计算哪一个存储最合适, 以及入库后的子目录, 可以考虑如下因素: 1. 批次 1. 所需要的存储量 1. 业务数据集内的属性 :param ib_option: :param need_storage_size: 所需的存储量 :param ib_batch_no: 批次编号 :param ib_schema: 入库模式 :param dataset_xml: 业务数据集的xml :return: dest_ib_storage_id: 目标入库的存储标识, 如果没有满足要求的, 则返回None dest_ib_root_path: 目标入库的存储的目录 desc_ib_dir_id: 入库的父目录的标识, 默认可以等于目标存储的标识 dest_ib_subpath: 入库的数据的子目录, 如果没有满足要求的, 则返回None message: 错误消息 """ storage_id = '' storage_title = '' storage_root_path = '' storage_volumn_free = 0 storage_volumn_max = 0 ib_schema_storage_type = CJson.dict_attr_by_path( ib_schema, 'storage.type', self.InBound_Storage_Match_Mode_Auto) if CUtils.equal_ignore_case(ib_schema_storage_type, self.InBound_Storage_Match_Mode_Set): storage_id = CJson.dict_attr_by_path(ib_schema, 'storage.id', '') if not CUtils.equal_ignore_case(storage_id, ''): sql_storage = ''' select dm2_storage.dstid, dm2_storage.dsttitle, coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) as root_path, dm2_storage.dst_volumn_max, dm2_storage.dst_volumn_max - coalesce(stat.file_size_sum, 0) as free_space from dm2_storage left join ( select dsfstorageid, sum(dsffilesize) as file_size_sum from dm2_storage_file left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid where dm2_storage.dsttype = 'core' group by dsfstorageid ) stat on dm2_storage.dstid = stat.dsfstorageid where dm2_storage.dsttype = 'core' and dm2_storage.dstid = :storage_id ''' ds_available_storage = CFactory().give_me_db( self.get_mission_db_id()).one_row( sql_storage, {'storage_id': storage_id}) if ds_available_storage.is_empty(): return None, None, None, None, '没有找到编号为[{0}]的存储! '.format( storage_id) else: storage_volumn_max = ds_available_storage.value_by_name( 0, 'dst_volumn_max', 0) storage_volumn_free = ds_available_storage.value_by_name( 0, 'free_space', 0) storage_title = ds_available_storage.value_by_name( 0, 'dsttitle', '') if storage_volumn_max > 0 and storage_volumn_free < need_storage_size: CLogger().debug( '存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 本批数据无法入库到该存储下!' .format(storage_title, storage_volumn_free, need_storage_size)) return \ None, None, None, None, \ '存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 本批数据无法入库到该存储下!'.format( storage_title, storage_volumn_free, need_storage_size ) storage_root_path = ds_available_storage.value_by_name( 0, 'root_path', '') if CUtils.equal_ignore_case(storage_id, ''): sql_available_storage = ''' select dm2_storage.dstid, dm2_storage.dsttitle, coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) as root_path, dm2_storage.dst_volumn_max, dm2_storage.dst_volumn_max - coalesce(stat.file_size_sum, 0) as free_space from dm2_storage left join ( select dsfstorageid, sum(dsffilesize) as file_size_sum from dm2_storage_file left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid where dm2_storage.dsttype = 'core' group by dsfstorageid ) stat on dm2_storage.dstid = stat.dsfstorageid where dm2_storage.dsttype = 'core' order by dm2_storage.dst_volumn_max desc ''' ds_available_storage = CFactory().give_me_db( self.get_mission_db_id()).all_row(sql_available_storage) for storage_index in range(0, ds_available_storage.size()): storage_title = ds_available_storage.value_by_name( storage_index, 'dsttitle', '') storage_volumn_max = ds_available_storage.value_by_name( storage_index, 'dst_volumn_max', 0) storage_volumn_free = ds_available_storage.value_by_name( storage_index, 'free_space', 0) if storage_volumn_max <= 0: storage_id = ds_available_storage.value_by_name( storage_index, 'dstid', '') storage_root_path = ds_available_storage.value_by_name( storage_index, 'root_path', '') CLogger().debug('存储[{0}]的存储空间没有限制, 系统将把本批数据入库到该存储下'.format( storage_title)) break elif storage_volumn_free > need_storage_size: storage_id = ds_available_storage.value_by_name( storage_index, 'dstid', '') storage_root_path = ds_available_storage.value_by_name( storage_index, 'root_path', '') CLogger().debug( '存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 系统将把本批数据入库到该存储下' .format(storage_title, storage_volumn_free, need_storage_size)) break if CUtils.equal_ignore_case(storage_id, ''): return \ None, None, None, None, \ '本批待入库数据的容量为[{0}], 未找到符合要求的存储, 请检查各个存储的剩余空间!'.format(need_storage_size) # 当前字典将用于计算入库到存储中的相对子目录 params = dict() # 将批次信息加入字典 params['batch_id'] = ib_batch_no # 将待入库数据集的元数据信息(一级元素)加入字典 self.metadata_bus_2_params(dataset_xml, params) # 开始用字典重算相对子目录 try: ib_schema_storage_path = CUtils.replace_placeholder( CUtils.any_2_str(CJson.dict_attr_by_path( ib_schema, 'path', '')), params, False) except Exception as error: return \ None, None, None, None, \ '本批待入库数据的入库目标目录无法确认, 部分信息未提供, 具体错误信息: [{0}]!'.format(error.__str__()) if storage_volumn_max <= 0: return \ storage_id, storage_root_path, storage_id, ib_schema_storage_path, \ '存储[{0}]的存储空间无限制, 本批数据存储所需空间为[{1}], 系统将把本批数据入库到该存储下'.format( storage_title, need_storage_size ) else: return \ storage_id, storage_root_path, storage_id, ib_schema_storage_path, \ '存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 系统将把本批数据入库到该存储下'.format( storage_title, storage_volumn_free, need_storage_size )
def __a_check_value_range__(cls, result_template, value, title_prefix, qa_items): """ 判断数字的范围,如经纬度坐标值:(-180 ~ 180) (-90~90) @param result_template: @param value: @param title_prefix: @param qa_items: @return: """ default_value = -1.000001 range_max = CUtils.to_decimal( CJson.dict_attr_by_path( qa_items, '{0}.{1}'.format(cls.Name_Range, cls.Name_Max), default_value)) range_min = CUtils.to_decimal( CJson.dict_attr_by_path( qa_items, '{0}.{1}'.format(cls.Name_Range, cls.Name_Min), default_value)) result_dict = copy.deepcopy(result_template) value_real = CUtils.to_decimal(value, -1) if range_max != default_value and range_min != default_value: if range_min <= value_real <= range_max: result_dict[ cls. Name_Message] = '[{0}]的值[{1}]在指定的[{2}-{3}]范围内, 符合要求!'.format( title_prefix, value, range_min, range_max) result_dict[cls.Name_Result] = cls.QA_Result_Pass else: result_dict[ cls. Name_Message] = '[{0}]的值[{1}]在指定的[{2}-{3}]范围外, 请检查!'.format( title_prefix, value, range_min, range_max) elif range_min != default_value: if range_min <= value_real: result_dict[ cls.Name_Message] = '[{0}]的值[{1}]大于最小值[{2}], 符合要求!'.format( title_prefix, value, range_min) result_dict[cls.Name_Result] = cls.QA_Result_Pass else: result_dict[ cls.Name_Message] = '[{0}]的值[{1}]低于最小值[{2}], 请检查!'.format( title_prefix, value, range_min) elif range_max != default_value: if range_max >= value_real: result_dict[ cls.Name_Message] = '[{0}]的值[{1}]低于最大值[{2}], 符合要求!'.format( title_prefix, value, range_max) result_dict[cls.Name_Result] = cls.QA_Result_Pass else: result_dict[ cls.Name_Message] = '[{0}]的值[{1}]超过最大值[{2}], 请检查!'.format( title_prefix, value, range_max) else: result_dict[ cls.Name_Message] = '[{0}]的值[{1}]未给定限定范围, 默认符合要求!'.format( title_prefix, value) result_dict[cls.Name_Result] = cls.QA_Result_Pass return result_dict