def __a_check_file__(cls, result_template: dict, file_name_with_path: str,
qa_items: dict) -> list:
"""
根据规则, 验证文件的合法性
完成 负责人 赵宇飞 在这里对文件的其他内容进行质检, 目前实现了检查文件大小, 请参考__a_check_file_size__完善其他内容, 包括并不限于
验证文件可读性, 验证元数据文件可读性, 验证元数据文件的格式(xml\json), 以便于后面在处理元数据时, 不会出现异常
:param result_template 检查结果的模板
:param file_name_with_path: 文件名
:param qa_items: 检查项目, keywords
:return:
"""
result_list = list()
result_list.append(
cls.__a_check_file_size__(
result_template, file_name_with_path,
CJson.dict_attr_by_path(
qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Min),
-1),
CJson.dict_attr_by_path(
qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Max),
-1)))
result_list.append(
cls.__a_check_file_format__(
result_template, file_name_with_path,
CJson.dict_attr_by_path(qa_items, cls.Name_Format, None)))
return result_list
def get_dest_storage(self, ib_batch_no, need_storage_size, ib_option,
ib_schema, dataset_xml):
"""
计算哪一个存储最合适, 以及入库后的子目录, 可以考虑如下因素:
1. 批次
1. 所需要的存储量
1. 业务数据集内的属性
:param ib_option:
:param need_storage_size: 所需的存储量
:param ib_batch_no: 批次编号
:param ib_schema: 入库模式
:param dataset_xml: 业务数据集的xml
:return:
dest_ib_storage_id: 目标入库的存储标识, 如果没有满足要求的, 则返回None
dest_ib_root_path: 目标入库的存储的目录
desc_ib_dir_id: 入库的父目录的标识, 默认可以等于目标存储的标识
dest_ib_subpath: 入库的数据的子目录, 如果没有满足要求的, 则返回None
message: 错误消息
"""
storage_id = ''
storage_title = ''
storage_root_path = ''
storage_volumn_free = 0
storage_volumn_max = 0
ib_schema_storage_type = CJson.dict_attr_by_path(
ib_schema, 'storage.type', self.InBound_Storage_Match_Mode_Auto)
if CUtils.equal_ignore_case(ib_schema_storage_type,
self.InBound_Storage_Match_Mode_Set):
storage_id = CJson.dict_attr_by_path(ib_schema, 'storage.id', '')
if not CUtils.equal_ignore_case(storage_id, ''):
sql_storage = '''
select dm2_storage.dstid, dm2_storage.dsttitle,
coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) as root_path,
dm2_storage.dst_volumn_max, dm2_storage.dst_volumn_max - coalesce(stat.file_size_sum, 0) as free_space
from dm2_storage left join (
select dsfstorageid, sum(dsffilesize) as file_size_sum
from dm2_storage_file left join dm2_storage
on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where dm2_storage.dsttype = 'core'
group by dsfstorageid
) stat on dm2_storage.dstid = stat.dsfstorageid
where dm2_storage.dsttype = 'core' and dm2_storage.dstid = :storage_id
'''
ds_available_storage = CFactory().give_me_db(
self.get_mission_db_id()).one_row(
sql_storage, {'storage_id': storage_id})
if ds_available_storage.is_empty():
return None, None, None, None, '没有找到编号为[{0}]的存储! '.format(
storage_id)
else:
storage_volumn_max = ds_available_storage.value_by_name(
0, 'dst_volumn_max', 0)
storage_volumn_free = ds_available_storage.value_by_name(
0, 'free_space', 0)
storage_title = ds_available_storage.value_by_name(
0, 'dsttitle', '')
if storage_volumn_max > 0 and storage_volumn_free < need_storage_size:
CLogger().debug(
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 本批数据无法入库到该存储下!'
.format(storage_title, storage_volumn_free,
need_storage_size))
return \
None, None, None, None, \
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 本批数据无法入库到该存储下!'.format(
storage_title, storage_volumn_free, need_storage_size
)
storage_root_path = ds_available_storage.value_by_name(
0, 'root_path', '')
if CUtils.equal_ignore_case(storage_id, ''):
sql_available_storage = '''
select dm2_storage.dstid, dm2_storage.dsttitle,
coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) as root_path,
dm2_storage.dst_volumn_max, dm2_storage.dst_volumn_max - coalesce(stat.file_size_sum, 0) as free_space
from dm2_storage left join (
select dsfstorageid, sum(dsffilesize) as file_size_sum
from dm2_storage_file left join dm2_storage
on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where dm2_storage.dsttype = 'core'
group by dsfstorageid
) stat on dm2_storage.dstid = stat.dsfstorageid
where dm2_storage.dsttype = 'core'
order by dm2_storage.dst_volumn_max desc
'''
ds_available_storage = CFactory().give_me_db(
self.get_mission_db_id()).all_row(sql_available_storage)
for storage_index in range(0, ds_available_storage.size()):
storage_title = ds_available_storage.value_by_name(
storage_index, 'dsttitle', '')
storage_volumn_max = ds_available_storage.value_by_name(
storage_index, 'dst_volumn_max', 0)
storage_volumn_free = ds_available_storage.value_by_name(
storage_index, 'free_space', 0)
if storage_volumn_max <= 0:
storage_id = ds_available_storage.value_by_name(
storage_index, 'dstid', '')
storage_root_path = ds_available_storage.value_by_name(
storage_index, 'root_path', '')
CLogger().debug('存储[{0}]的存储空间没有限制, 系统将把本批数据入库到该存储下'.format(
storage_title))
break
elif storage_volumn_free > need_storage_size:
storage_id = ds_available_storage.value_by_name(
storage_index, 'dstid', '')
storage_root_path = ds_available_storage.value_by_name(
storage_index, 'root_path', '')
CLogger().debug(
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 系统将把本批数据入库到该存储下'
.format(storage_title, storage_volumn_free,
need_storage_size))
break
if CUtils.equal_ignore_case(storage_id, ''):
return \
None, None, None, None, \
'本批待入库数据的容量为[{0}], 未找到符合要求的存储, 请检查各个存储的剩余空间!'.format(need_storage_size)
# 当前字典将用于计算入库到存储中的相对子目录
params = dict()
# 将批次信息加入字典
params['batch_id'] = ib_batch_no
# 将待入库数据集的元数据信息(一级元素)加入字典
self.metadata_bus_2_params(dataset_xml, params)
# 开始用字典重算相对子目录
try:
ib_schema_storage_path = CUtils.replace_placeholder(
CUtils.any_2_str(CJson.dict_attr_by_path(
ib_schema, 'path', '')), params, False)
except Exception as error:
return \
None, None, None, None, \
'本批待入库数据的入库目标目录无法确认, 部分信息未提供, 具体错误信息: [{0}]!'.format(error.__str__())
if storage_volumn_max <= 0:
return \
storage_id, storage_root_path, storage_id, ib_schema_storage_path, \
'存储[{0}]的存储空间无限制, 本批数据存储所需空间为[{1}], 系统将把本批数据入库到该存储下'.format(
storage_title, need_storage_size
)
else:
return \
storage_id, storage_root_path, storage_id, ib_schema_storage_path, \
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 系统将把本批数据入库到该存储下'.format(
storage_title, storage_volumn_free, need_storage_size
)
def __a_check_value_range__(cls, result_template, value, title_prefix,
qa_items):
"""
判断数字的范围，如经纬度坐标值：（-180 ~ 180） （-90~90）
@param result_template:
@param value:
@param title_prefix:
@param qa_items:
@return:
"""
default_value = -1.000001
range_max = CUtils.to_decimal(
CJson.dict_attr_by_path(
qa_items, '{0}.{1}'.format(cls.Name_Range, cls.Name_Max),
default_value))
range_min = CUtils.to_decimal(
CJson.dict_attr_by_path(
qa_items, '{0}.{1}'.format(cls.Name_Range, cls.Name_Min),
default_value))
result_dict = copy.deepcopy(result_template)
value_real = CUtils.to_decimal(value, -1)
if range_max != default_value and range_min != default_value:
if range_min <= value_real <= range_max:
result_dict[
cls.
Name_Message] = '[{0}]的值[{1}]在指定的[{2}-{3}]范围内, 符合要求!'.format(
title_prefix, value, range_min, range_max)
result_dict[cls.Name_Result] = cls.QA_Result_Pass
else:
result_dict[
cls.
Name_Message] = '[{0}]的值[{1}]在指定的[{2}-{3}]范围外, 请检查!'.format(
title_prefix, value, range_min, range_max)
elif range_min != default_value:
if range_min <= value_real:
result_dict[
cls.Name_Message] = '[{0}]的值[{1}]大于最小值[{2}], 符合要求!'.format(
title_prefix, value, range_min)
result_dict[cls.Name_Result] = cls.QA_Result_Pass
else:
result_dict[
cls.Name_Message] = '[{0}]的值[{1}]低于最小值[{2}], 请检查!'.format(
title_prefix, value, range_min)
elif range_max != default_value:
if range_max >= value_real:
result_dict[
cls.Name_Message] = '[{0}]的值[{1}]低于最大值[{2}], 符合要求!'.format(
title_prefix, value, range_max)
result_dict[cls.Name_Result] = cls.QA_Result_Pass
else:
result_dict[
cls.Name_Message] = '[{0}]的值[{1}]超过最大值[{2}], 请检查!'.format(
title_prefix, value, range_max)
else:
result_dict[
cls.Name_Message] = '[{0}]的值[{1}]未给定限定范围, 默认符合要求!'.format(
title_prefix, value)
result_dict[cls.Name_Result] = cls.QA_Result_Pass
return result_dict