def __a_check_file__(cls, result_template: dict, file_name_with_path: str,
qa_items: dict) -> list:
"""
根据规则, 验证文件的合法性
完成 负责人 赵宇飞 在这里对文件的其他内容进行质检, 目前实现了检查文件大小, 请参考__a_check_file_size__完善其他内容, 包括并不限于
验证文件可读性, 验证元数据文件可读性, 验证元数据文件的格式(xml\json), 以便于后面在处理元数据时, 不会出现异常
:param result_template 检查结果的模板
:param file_name_with_path: 文件名
:param qa_items: 检查项目, keywords
:return:
"""
result_list = list()
result_list.append(
cls.__a_check_file_size__(
result_template, file_name_with_path,
CJson.dict_attr_by_path(
qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Min),
-1),
CJson.dict_attr_by_path(
qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Max),
-1)))
result_list.append(
cls.__a_check_file_format__(
result_template, file_name_with_path,
CJson.dict_attr_by_path(qa_items, cls.Name_Format, None)))
return result_list
def get_extent_by_vectorlayer(self, layer, feature_count) -> CJson:
"""
构建图层四至范围的json对象
@param layer:
@param feature_count:
@return:
"""
json_extent = CJson()
if feature_count == 0:
json_extent.set_value_of_name("minx", 0)
json_extent.set_value_of_name("maxx", 0)
json_extent.set_value_of_name("miny", 0)
json_extent.set_value_of_name("maxy", 0)
else:
extent = layer.GetExtent()
# print('extent:', extent)
if extent is not None:
# print('ul:', extent[0], extent[3])
# print('lr:', extent[1], extent[2])
json_extent.set_value_of_name("minx", extent[0])
json_extent.set_value_of_name("maxx", extent[1])
json_extent.set_value_of_name("miny", extent[2])
json_extent.set_value_of_name("maxy", extent[3])
extent = None
return json_extent
def create_view_json(self, params_json: CJson):
"""
设置为一个参数，供进程调用
"""
image_path = params_json.xpath_one('image_path', None)
browse_full_path = params_json.xpath_one('browse_full_path', None)
thumb_full_path = params_json.xpath_one('thumb_full_path', None)
geotiff_full_path = params_json.xpath_one('geotiff_full_path', None)
result = self.create_view(image_path, browse_full_path, thumb_full_path, geotiff_full_path)
return result
def get_sync_dict_list(self, insert_or_updata) -> list:
"""
insert_or_updata 指明配置的是更新还是插入，-1时为插入，0为更新
本方法的写法为强规则，调用add_value_to_sync_dict_list配置
第一个参数为list，第二个参数为字段名，第三个参数为字段值，第四个参数为特殊配置
"""
sync_dict_list = self.get_sync_predefined_dict_list(insert_or_updata)
object_table_data = self._dataset
# 时间信息
dso_time = object_table_data.value_by_name(0, 'dso_time', '')
dso_time_json = CJson()
dso_time_json.load_obj(dso_time)
self.add_value_to_sync_dict_list(
sync_dict_list, 'begdate',
dso_time_json.xpath_one('start_time', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'enddate', dso_time_json.xpath_one('end_time', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'imagedate',
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', '')))
self.add_value_to_sync_dict_list(
sync_dict_list, 'producetime',
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', '')))
self.add_value_to_sync_dict_list(
sync_dict_list, 'imagedatetag',
self.transform_time_to_imagedatetag(
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', ''))))
if insert_or_updata:
self.add_value_to_sync_dict_list(sync_dict_list, 'isdel', '1')
return sync_dict_list
def process(self) -> str:
"""
完成 负责人 张源博、赵宇飞 在这里提取影像数据的快视图, 将元数据文件存储在self.file_content.view_root_dir下
注意返回的串中有快视图和拇指图的文件名
注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!!
:return:
"""
# 获取对象类型
type = 'default'
group = 'default'
catalog = 'default'
# 构建数据对象object对应的识别插件，获取get_information里面的信息
class_classified_obj = CObject.get_plugins_instance_by_object_id(self.file_info.db_server_id, self.object_id)
if class_classified_obj is not None:
plugins_info = class_classified_obj.get_information()
type = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Type, 'default')
group = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Group, 'default')
catalog = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Catalog, 'default')
create_time = CTime.today()
create_format_time = CTime.format_str(create_time, '%Y%m%d')
year = CTime.format_str(create_time, '%Y')
month = CTime.format_str(create_time, '%m')
day = CTime.format_str(create_time, '%d')
sep = CFile.sep() # 操作系统的不同处理分隔符不同
sep_list = [catalog, group, type, year, month, day]
relative_path_part = sep.join(sep_list) # 相对路径格式
view_relative_path_browse = r'{2}{0}{2}{1}_browse.png'.format(relative_path_part, self.object_id, sep)
view_relative_path_thumb = r'{2}{0}{2}{1}_thumb.jpg'.format(relative_path_part, self.object_id, sep)
view_relative_path_geotiff = r'{2}{0}{2}{1}_browse.tiff'.format(relative_path_part, self.object_id, sep)
browse_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_browse)
thumb_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_thumb)
geotiff_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_geotiff)
# 进程调用模式
json_out_view = CJson()
json_out_view.set_value_of_name('image_path', self.file_info.file_name_with_full_path)
json_out_view.set_value_of_name('browse_full_path', browse_full_path)
json_out_view.set_value_of_name('thumb_full_path', thumb_full_path)
json_out_view.set_value_of_name('geotiff_full_path', geotiff_full_path)
result_view = CProcessUtils.processing_method(self.create_view_json, json_out_view)
# result_view = self.create_view(self.file_info.file_name_with_full_path, browse_full_path, thumb_full_path,
# geotiff_full_path)
# result_view = self.create_view_json(json_out_view)
if CResult.result_success(result_view):
result = CResult.merge_result(self.Success, '处理完毕!')
result = CResult.merge_result_info(result, self.Name_Browse, view_relative_path_browse)
result = CResult.merge_result_info(result, self.Name_Thumb, view_relative_path_thumb)
result = CResult.merge_result_info(result, self.Name_Browse_GeoTiff, view_relative_path_geotiff)
else:
result = result_view
return result
def merge_result(cls, result, message=None, base=None) -> str:
new_result = CJson()
if base is not None:
new_result.load_json_text(base)
new_result.set_value_of_name(cls.Name_Result, result)
if message is not None:
new_result.set_value_of_name(cls.Name_Message, message)
return new_result.to_json()
def white_black_valid(self):
"""
检查指定文件是否符合白名单, 黑名单验证
"""
ds_storage_option = self._ds_storage.value_by_name(
0, 'dstotheroption', None)
if ds_storage_option == '' or ds_storage_option is None:
return True
dir_filter_white_list = CJson.json_attr_value(
ds_storage_option, self.Path_SO_Inbound_Filter_Dir_WhiteList, '')
dir_filter_black_list = CJson.json_attr_value(
ds_storage_option, self.Path_SO_Inbound_Filter_Dir_BlackList, '')
file_filter_white_list = CJson.json_attr_value(
ds_storage_option, self.Path_SO_Inbound_Filter_File_WhiteList, '')
file_filter_black_list = CJson.json_attr_value(
ds_storage_option, self.Path_SO_Inbound_Filter_File_BlackList, '')
result = True
if self.file_type != self.FileType_Unknown:
if (dir_filter_white_list != '') and (dir_filter_black_list != ''):
result = CFile.file_match(
self.file_path_with_rel_path,
dir_filter_white_list) and (not CFile.file_match(
self.file_path_with_rel_path, dir_filter_black_list))
elif dir_filter_white_list != '':
result = CFile.file_match(self.file_path_with_rel_path,
dir_filter_white_list)
elif dir_filter_black_list != '':
result = not CFile.file_match(self.file_path_with_rel_path,
dir_filter_black_list)
if not result:
return result
if self.file_type == self.FileType_File:
if (file_filter_white_list != '') and (file_filter_black_list !=
''):
return CFile.file_match(
self.file_name_without_path,
file_filter_white_list) and (not CFile.file_match(
self.file_name_without_path, file_filter_black_list))
elif file_filter_white_list != '':
return CFile.file_match(self.file_name_without_path,
file_filter_white_list)
elif file_filter_black_list != '':
return not CFile.file_match(self.file_name_without_path,
file_filter_black_list)
else:
return True
else:
return True
def process(self) -> str:
"""
"""
browse_full_path = CFile.join_file(self.view_path, '{0}_browse.png'.format(self.object_id))
thumb_full_path = CFile.join_file(self.view_path, '{0}_thumb.jpg'.format(self.object_id))
geotiff_full_path = CFile.join_file(self.view_path, '{0}_browse.tiff'.format(self.object_id))
# 进程调用模式
json_out_view = CJson()
json_out_view.set_value_of_name('image_path', self.transform_file)
json_out_view.set_value_of_name('browse_full_path', browse_full_path)
json_out_view.set_value_of_name('thumb_full_path', thumb_full_path)
json_out_view.set_value_of_name('geotiff_full_path', geotiff_full_path)
result_view = CProcessUtils.processing_method(self.create_view_json, json_out_view)
# result_view = self.create_view(self.file_info.file_name_with_full_path, browse_full_path, thumb_full_path,
# geotiff_full_path)
# result_view = self.create_view_json(json_out_view)
if CResult.result_success(result_view):
# 清理不必要的文件
delect_file_list = list()
delect_file_list.append('{0}.aux.xml'.format(browse_full_path))
delect_file_list.append('{0}.aux.xml'.format(thumb_full_path))
delect_file_list.append(geotiff_full_path)
for delect_file in delect_file_list:
if CFile.file_or_path_exist(delect_file):
CFile.remove_file(delect_file)
result = CResult.merge_result(self.Success, '处理完毕!')
result = CResult.merge_result_info(result, self.Name_Browse, CFile.file_name(browse_full_path))
result = CResult.merge_result_info(result, self.Name_Thumb, CFile.file_name(thumb_full_path))
result = CResult.merge_result_info(result, self.Name_Browse_GeoTiff, CFile.file_name(geotiff_full_path))
else:
# 清理不必要的文件
delect_file_list = list()
delect_file_list.append(browse_full_path)
delect_file_list.append(thumb_full_path)
delect_file_list.append('{0}.aux.xml'.format(browse_full_path))
delect_file_list.append('{0}.aux.xml'.format(thumb_full_path))
delect_file_list.append(geotiff_full_path)
for delect_file in delect_file_list:
if CFile.file_or_path_exist(delect_file):
CFile.remove_file(delect_file)
result = result_view
return result
def a_json_element(cls, audit_id, audit_title, audit_group, audit_result,
json_obj: CJson, xpath: str, qa_items: dict) -> list:
"""
判断一个json元数据中, 指定的jsonpath, 对应的element, 满足qa_items参数中的检测项目
:param audit_id:
:param audit_title:
:param audit_group:
:param audit_result:
:param json_obj:
:param xpath:
:param qa_items:
:return:
"""
result_dict = cls.__init_audit_dict__(audit_id, audit_title,
audit_group, audit_result)
if json_obj is None:
result_dict[cls.Name_Message] = 'Json对象不合法, 节点[{0}]不存在'.format(
xpath)
return [result_dict]
json_value = json_obj.xpath_one(xpath, None)
if json_value is not None:
return cls.__a_check_value__(result_dict, json_value,
'属性[{0}]'.format(audit_title),
qa_items)
else:
result_dict[cls.Name_Message] = 'Json对象的节点[{0}]不存在, 请检查修正!'.format(
xpath)
return [result_dict]
def set_value_from_file(self, file_name: str, file_format: str,
file_encoding: str):
"""
设置value为文件内容
. 根据文件格式, 可以加载文件内容
. 如果文件格式为二进制, 则value存储文件名
:param file_name:
:param file_format:
:param file_encoding:
:return:
"""
if CUtils.equal_ignore_case(file_format, CResource.FileFormat_TXT):
self._value = {
CResource.Name_Text: CFile.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_XML):
self._value = {
CResource.Name_Text: CXml.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_Json):
self._value = {
CResource.Name_Text: CJson.file_2_str(file_name),
CResource.Name_Type: CResource.DataValueType_SQL
}
else:
self._value = {
CResource.Name_Text: file_name,
CResource.Name_Type: CResource.DataValueType_File
}
def test_get_attr(self):
json = CJson()
json.load_json_text(self.test_text)
assert json.xpath_one('a', -1) == 1
assert json.xpath_one('aa', -1) == -1
assert json.xpath_one('student[0].name', '') == '小明'
assert json.xpath_one('student[1].name', '') == '小王'
def set_metadata_bus(self, metadata_bus_extract_result: int,
metadata_bus_extract_memo: str,
metadata_bus_type: int, metadata_bus_text):
self.__metadata_bus_extract_result__ = metadata_bus_extract_result
self.__metadata_bus_extract_memo__ = metadata_bus_extract_memo
self.__metadata_bus_type__ = metadata_bus_type
if self.__metadata_bus_type__ == self.MetaDataFormat_Json:
self.__metadata_bus_text__ = ''
self.__metadata_bus_xml__ = CXml()
self.__metadata_bus_json__.load_json_text(metadata_bus_text)
elif self.__metadata_bus_type__ == self.MetaDataFormat_XML:
self.__metadata_bus_text__ = ''
self.__metadata_bus_xml__.load_xml(metadata_bus_text)
self.__metadata_bus_json__ = CJson()
else:
self.__metadata_bus_text__ = metadata_bus_text
self.__metadata_bus_xml__ = CXml()
self.__metadata_bus_json__ = CJson()
def set_metadata_bus_file(self, metadata_bus_extract_result: int,
metadata_bus_extract_memo: str,
metadata_type: int, file_name):
self.__metadata_bus_extract_result__ = metadata_bus_extract_result
self.__metadata_bus_extract_memo__ = metadata_bus_extract_memo
self.__metadata_bus_type__ = metadata_type
if self.__metadata_bus_type__ == self.MetaDataFormat_Json:
self.__metadata_bus_text__ = ''
self.__metadata_bus_xml__ = CXml()
self.__metadata_bus_json__.load_file(file_name)
elif self.__metadata_bus_type__ == self.MetaDataFormat_XML:
self.__metadata_bus_text__ = ''
self.__metadata_bus_xml__.load_file(file_name)
self.__metadata_bus_json__ = CJson()
else:
self.__metadata_bus_text__ = CFile.file_2_str(file_name)
self.__metadata_bus_xml__ = CXml()
self.__metadata_bus_json__ = CJson()
def params_value_by_name(self, attr_name: str, default_value):
"""
通过解析传入参数, 直接获取任务执行方面的参数, 该参数都存储在trigger对象下
:param attr_name:
:param default_value:
:return:
"""
return CJson().json_attr_value(
self.__schedule_params__,
'{0}.{1}'.format(self.TRIGGER_Params, attr_name), default_value)
def params_value_by_name(self, params: str, attr_name: str, default_value):
"""
通过解析传入参数, 直接获取任务执行方面的参数, 该参数都存储在job对象下
:param params:
:param attr_name:
:param default_value:
:return:
"""
return CJson().json_attr_value(
params, '{0}.{1}'.format(self.Name_Process, attr_name),
default_value)
def register_dm_metadata_plugins(self):
sql_register_dm_metadata_plugins_clear = '''
truncate table dm2_storage_object_def cascade
'''
sql_unregister_dm_metadata_plugins = '''
delete from dm2_storage_object_def where dsodid = :dsodid
'''
sql_register_dm_metadata_plugins = '''
insert into dm2_storage_object_def(
dsodid, dsodtitle, dsodtype, dsodtypetitle, dsodtypecode, dsodgroup, dsodgrouptitle,
dsodcatalog, dsodcatalogtitle, dsod_otheroption)
values (:dsodid, :dsodtitle, :dsodtype, :dsodtypetitle, :dsodtypecode, :dsodgroup, :dsodgrouptitle,
:dsodcatalog, :dsodcatalogtitle, :dsod_otheroption)
'''
CFactory().give_me_db().execute(sql_register_dm_metadata_plugins_clear)
plugins_root_dir = CSys.get_plugins_root_dir()
plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir)
for plugins_type in plugins_type_list:
if CFile.is_dir(CFile.join_file(
plugins_root_dir, plugins_type)) and (
not (str(plugins_type)).startswith('_')):
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), plugins_type)
path = CFile.join_file(CSys.get_plugins_root_dir(),
plugins_type)
plugins_file_list = CFile.file_or_subpath_of_path(
path, '{0}_*.{1}'.format(self.Name_Plugins,
self.FileExt_Py))
for file_name_without_path in plugins_file_list:
file_main_name = CFile.file_main_name(
file_name_without_path)
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, file_main_name, None)
plugins_info = class_classified_obj.get_information()
json_obj = CJson()
json_obj.set_value_of_name(
self.Name_Is_Spatial,
CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Is_Spatial,
False))
json_obj.set_value_of_name(
self.Name_Is_DataSet,
CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Is_Dataset,
False))
plugins_info['dsod_otheroption'] = json_obj.to_json()
print('{0}/{1}:{2}'.format(plugins_type, file_main_name,
plugins_info))
CFactory().give_me_db().execute(
sql_unregister_dm_metadata_plugins, plugins_info)
CFactory().give_me_db().execute(
sql_register_dm_metadata_plugins, plugins_info)
def __init__(self):
self.__quality__ = CQuality()
self.__metadata_extract_result__ = self.Not_Support
self.__metadata_extract_memo__ = ''
self.__metadata_text__ = None
self.__metadata_xml__ = CXml()
self.__metadata_json__ = CJson()
self.__metadata_type__ = self.MetaDataFormat_Text
self.__metadata_bus_extract_result__ = self.Not_Support
self.__metadata_bus_extract_memo__ = ''
self.__metadata_bus_text__ = None
self.__metadata_bus_xml__ = CXml()
self.__metadata_bus_json__ = CJson()
self.__metadata_bus_type__ = self.MetaDataFormat_Text
self.__thumb_img_file_name__ = ''
self.__browse_img_file_name__ = ''
self.__metadata_view_extract_result__ = self.Not_Support
self.__metadata_view_extract_memo__ = ''
self.__time_information__ = CJson()
self.__metadata_time_extract_result__ = self.Not_Support
self.__metadata_time_extract_memo__ = ''
self.__metadata_spatial_extract_result__ = self.Not_Support
self.__metadata_spatial_extract_memo__ = ''
self.__metadata_spatial__ = CMDSpatial()
def summary(self) -> str:
json_obj = CJson()
json_obj.set_value_of_name(
self.Name_Total,
self.__quality_result_of_level(self.__XPath_Total))
metadata_qa_s = {
self.Name_Data:
self.__quality_result_of_level(self.__XPath_MetaData_Data),
self.Name_Business:
self.__quality_result_of_level(self.__XPath_MetaData_Bus)
}
json_obj.set_value_of_name(self.Name_MetaData, metadata_qa_s)
json_obj.set_value_of_name(
self.Name_Data, {
self.Name_Items:
self.__quality_result_of_level(self.__XPath_Data_Items)
})
return json_obj.to_json()
def params_value_by_name(self, attr_name: str, default_value):
"""
通过解析传入参数, 直接获取任务执行方面的参数, 该参数都存储在job对象下
:param attr_name:
:param default_value:
:return:
"""
if self.__params is None:
return default_value
else:
return CJson.json_attr_value(
self.__params, '{0}.{1}'.format(self.NAME_JOB, attr_name),
default_value)
def process(self) -> str:
"""
在这里提取矢量数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式
:return:
"""
result = super().process()
ds_metadata = CFactory().give_me_db(
self.file_info.db_server_id).one_row(
'''
select dsometadatatext, dsometadatajson, dsometadataxml
from dm2_storage_object
where dsoid = :object_id
''', {'object_id': self.object_id})
metadata_filename = CFile.join_file(
self.file_content.work_root_dir,
'{0}.metadata'.format(self.object_name))
text_metadata = ds_metadata.value_by_name(0, 'dsometadatatext', None)
json_metadata = ds_metadata.value_by_name(0, 'dsometadatajson', None)
xml_metadata = ds_metadata.value_by_name(0, 'dsometadataxml', None)
format_metadata = self.MetaDataFormat_Text
if json_metadata is not None:
format_metadata = self.MetaDataFormat_Json
CJson.str_2_file(json_metadata, metadata_filename)
elif xml_metadata is not None:
format_metadata = self.MetaDataFormat_XML
CXml.str_2_file(xml_metadata, metadata_filename)
elif text_metadata is not None:
format_metadata = self.MetaDataFormat_XML
CFile.str_2_file(text_metadata, metadata_filename)
else:
return result
result = CResult.merge_result_info(result, self.Name_FileName,
metadata_filename)
return CResult.merge_result_info(result, self.Name_Format,
format_metadata)
def get_other_metadata_by_raster(self, other_metadata: dict) -> CJson:
"""
获取栅格文件的subdatasets、geolocation、rpc元数据信息
:param other_metadata:
:return:
"""
other_data = CJson()
other_data.set_value_of_name('valid', True)
other_metadata_list = []
for i in other_metadata:
if other_metadata[i] is not None:
sub_metadata_item = other_metadata[i]
other_metadata_list.append(sub_metadata_item)
other_data.set_value_of_name('metadata', other_metadata_list)
return other_data
def get_geometry_by_vectorlayer(self, layer) -> CJson:
"""
构建图层图形类型的json对象
@param layer:
@return:
"""
json_geometry = CJson()
geomtype = layer.GetGeomType() # 数字
json_geometry.set_value_of_name("type", geomtype)
feature_count = layer.GetFeatureCount()
if feature_count > 0:
for i in range(feature_count):
feature = layer.GetFeature(i)
if feature is not None:
geom = feature.geometry()
if geom is not None:
geom_name = geom.GetGeometryName()
json_geometry.set_value_of_name("name", geom_name)
return json_geometry
return json_geometry
def get_sync_predefined_dict_list(self, insert_or_updata) -> list:
"""
insert_or_updata 指明配置的是更新还是插入，-1时为插入，0为更新
本方法的写法为强规则，调用add_value_to_sync_dict_list配置
第一个参数为list，第二个参数为字段名，第三个参数为字段值，第四个参数为特殊配置
本方法处理公共部分
datacount:数据量 secrecylevel:密级 regioncode:行政区码 regionname:行政区 resolution:分辨率
colormodel:色彩模式 piexldepth:像素位数 scale:比例尺分母 mainrssource:主要星源 交插件去处理
"""
sync_dict_list = list()
object_table_id = self._obj_id # 获取oid
object_table_data = self._dataset
self.add_value_to_sync_dict_list(sync_dict_list, 'aprid',
object_table_id)
self.add_value_to_sync_dict_list(
sync_dict_list, 'productname',
object_table_data.value_by_name(0, 'dsoobjectname', ''))
self.add_value_to_sync_dict_list(sync_dict_list, 'producttype',
self._obj_type_code)
self.add_value_to_sync_dict_list(
sync_dict_list, 'dsodatatype',
object_table_data.value_by_name(0, 'dsodatatype', ''))
dso_time = object_table_data.value_by_name(0, 'dso_time', '')
dso_time_json = CJson()
dso_time_json.load_obj(dso_time)
self.add_value_to_sync_dict_list(
sync_dict_list, 'begdate',
dso_time_json.xpath_one('start_time', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'enddate', dso_time_json.xpath_one('end_time', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'imagedate',
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', '')))
# datacount:数据数量
# secrecylevel:密级
# regioncode:行政区码
# regionname:行政区 上面四个字段交插件处理
self.add_value_to_sync_dict_list( # 配置子查询，调用函数
sync_dict_list, 'centerx', '''
(select
st_x(st_centroid(
(select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}')
))::decimal(8, 2))
'''.format(object_table_id), self.DataValueType_SQL)
self.add_value_to_sync_dict_list(
sync_dict_list, 'centery', '''
(select
st_y(st_centroid(
(select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}')
))::decimal(8, 2))
'''.format(object_table_id), self.DataValueType_SQL)
self.add_value_to_sync_dict_list(
sync_dict_list, 'geomwkt', '''
st_astext(
(select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}')
)
'''.format(object_table_id), self.DataValueType_SQL)
self.add_value_to_sync_dict_list(
sync_dict_list, 'geomobj', '''
(select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}')
'''.format(object_table_id), self.DataValueType_SQL)
self.add_value_to_sync_dict_list(
sync_dict_list, 'browserimg',
object_table_data.value_by_name(0, 'dso_browser', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'thumbimg',
object_table_data.value_by_name(0, 'dso_thumb', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'producetime',
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', '')))
# resolution:分辨率，交插件处理
self.add_value_to_sync_dict_list(
sync_dict_list, 'imgsize', '''
(select round((sum(dodfilesize)/1048576),2) from dm2_storage_obj_detail where dodobjectid='{0}')
'''.format(object_table_id), self.DataValueType_SQL)
# colormodel:交插件处理
# piexldepth:交插件处理
if insert_or_updata:
self.add_value_to_sync_dict_list(sync_dict_list, 'isdel', '0')
now_time = CUtils.any_2_str(
datetime.datetime.now().strftime('%F %T'))
self.add_value_to_sync_dict_list(sync_dict_list, 'addtime',
now_time)
self.add_value_to_sync_dict_list(
sync_dict_list, 'extent',
"(select dso_geo_bb_wgs84 from dm2_storage_object where dsoid='{0}')"
.format(object_table_id), self.DataValueType_SQL)
self.add_value_to_sync_dict_list(
sync_dict_list, 'proj',
object_table_data.value_by_name(0, 'dso_prj_wkt', ''))
# remark:暂时为空
# ispublishservice:暂时为空
self.add_value_to_sync_dict_list(sync_dict_list, 'queryable', '0')
# scale:交插件处理
# mainrssource:交插件处理
self.add_value_to_sync_dict_list(
sync_dict_list, 'dsdid',
object_table_data.value_by_name(0, 'query_directory_id', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'dsfid',
object_table_data.value_by_name(0, 'query_file_id', ''))
self.add_value_to_sync_dict_list(
sync_dict_list, 'imagedatetag',
self.transform_time_to_imagedatetag(
CUtils.to_day_format(dso_time_json.xpath_one('time', ''),
dso_time_json.xpath_one('time', ''))))
return sync_dict_list
def execute(self) -> str:
inbound_ib_n_list = CFactory().give_me_db(
self.get_mission_db_id()).all_row('''
select
dsiid as query_ib_id
, dsiotheroption as query_ib_option
, dsidirectory as query_ib_relation_dir
, dsidirectoryid as query_ib_dir_id
, dsitargetstorageid as query_ib_target_storage_id
from dm2_storage_inbound
where dsi_na_status = {0}
'''.format(self.ProcStatus_WaitConfirm))
if inbound_ib_n_list.is_empty():
return CResult.merge_result(CResult.Success, '本次没有需要检查的通知任务！')
for data_index in range(inbound_ib_n_list.size()):
ds_ib_id = inbound_ib_n_list.value_by_name(data_index,
'query_ib_id', '')
ds_ib_option = CUtils.any_2_str(
inbound_ib_n_list.value_by_name(data_index, 'query_ib_option',
''))
ds_ib_directory_name = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_relation_dir', '')
ds_ib_directory_id = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_dir_id', '')
ds_ib_target_storage_id = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_target_storage_id', '')
module_name_list = CJson.json_attr_value(
ds_ib_option, self.Path_IB_Opt_Notify_module, None)
if module_name_list is None:
modules_root_dir = CSys.get_metadata_data_access_modules_root_dir(
)
module_file_list = CFile.file_or_subpath_of_path(
modules_root_dir,
'{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py))
module_name_list = list()
for module_file in module_file_list:
module_name_list.append(CFile.file_main_name(module_file))
CLogger().debug('正在检查入库批次[ds_ib_id]的通知进度...'.format(ds_ib_id))
try:
# 所有通知对象的统计数
sql_record_total_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
''', {
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_total_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_total_count, {'ib_id': ds_ib_id}, 0)
if record_total_count == 0:
self.update_inbound_na_result(
ds_ib_id,
CResult.merge_result(
self.Failure,
'入库任务下没有可通知的数据, 请检查异常情况! '.format(ds_ib_id)))
continue
# 已经完成的通知对象的统计数, 包括正常完成和错误的
sql_record_finished_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_notify_status in ({0}, {1})
and dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
'''.format(self.ProcStatus_Finished,
self.ProcStatus_Error),
{
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_finished_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_finished_count, {'ib_id': ds_ib_id}, 0)
# 错误的记录数
sql_record_error_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_notify_status = {0}
and dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
'''.format(self.ProcStatus_Error), {
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_error_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_error_count, {'ib_id': ds_ib_id}, 0)
if record_total_count != record_finished_count:
message = '入库任务[{0}]下的数据正在通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个...'.format(
ds_ib_id, record_total_count, record_finished_count,
record_error_count)
CLogger().debug(message)
self.update_inbound_na_progress(
ds_ib_id, CResult.merge_result(self.Failure, message))
else:
message = '入库任务[{0}]下的数据已经通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个, 请检查修正! '.format(
ds_ib_id, record_total_count, record_finished_count,
record_error_count)
CLogger().debug(message)
self.update_inbound_na_result(
ds_ib_id, CResult.merge_result(self.Success, message))
except Exception as error:
self.update_inbound_na_result(
ds_ib_id,
CResult.merge_result(
self.Failure,
'入库任务下的数据通知其他子系统过程中出现异常情况, 详细错误信息为: [{1}]'.format(
ds_ib_id, error.__str__())))
continue
return CResult.merge_result(self.Success, '本次通知监控任务成功结束！')
def to_file(cls, result_text, file_name):
return CJson.str_2_file(result_text, file_name)
def result_message(cls, result_text) -> str:
return CJson.json_attr_value(result_text, cls.Name_Message, '')
def result_info(cls, result_text, info_name: str, default_value):
return CJson.json_attr_value(result_text, info_name, default_value)
def result_success(cls, result_text) -> bool:
return CJson.json_attr_value(result_text, cls.Name_Result, cls.Failure) == cls.Success
def merge_result_info(cls, result_text, info_name: str, value):
return CJson.json_set_attr(result_text, info_name, value)
def _do_access(self) -> str:
try:
quality_info_xml = self._quality_info # 获取质检xml
quality_summary = self._dataset.value_by_name(0, 'dso_quality_summary', '')
quality_summary_json = CJson()
quality_summary_json.load_obj(quality_summary)
access_wait_flag = self.DB_False # 定义等待标志，为True则存在检查项目为等待
access_forbid_flag = self.DB_False # 定义禁止标志，为True则存在检查项目为禁止
message = ''
# 文件与影像质检部分
file_qa = quality_summary_json.xpath_one('total', '')
image_qa = quality_summary_json.xpath_one('metadata.data', '')
if CUtils.equal_ignore_case(file_qa, self.QA_Result_Error) \
or CUtils.equal_ignore_case(image_qa, self.QA_Result_Error):
message = message + '[数据与其相关文件的质检存在error!请进行修正！]'
access_forbid_flag = self.DB_True
elif CUtils.equal_ignore_case(file_qa, self.QA_Result_Warn) \
or CUtils.equal_ignore_case(image_qa, self.QA_Result_Warn):
message = message + '[数据与其相关文件的质检存在warn!请进行检查！]'
access_wait_flag = self.DB_True
elif CUtils.equal_ignore_case(quality_summary, ''):
message = message + '[数据质检未进行，可能数据存在问题!请进行检查！]'
access_forbid_flag = self.DB_True
else:
pass
for qa_name, qa_id in self.access_check_dict().items(): # 循环写好的检查列表
# qa_id = CUtils.dict_value_by_name(access_check_dict, 'qa_id', '') # 获取id
qa_node = quality_info_xml.xpath_one("//item[@id='{0}']".format(qa_id)) # 查询xml中的节点
if qa_node is not None:
node_result = CXml.get_attr(qa_node, self.Name_Result, '', False) # 获取质检结果
if CUtils.equal_ignore_case(node_result, self.QA_Result_Pass):
pass
elif CUtils.equal_ignore_case(node_result, self.QA_Result_Warn): # 警告则等待
message = message + '[业务元数据的质检中，项目{0}不符合要求，建议修正！]'.format(qa_name)
access_wait_flag = self.DB_True
else: # 错误以及其他情况，比如''，或者为其他字段
message = message + '[业务元数据的质检中，项目{0}不符合要求，必须修改后方可入库！]'.format(qa_name)
access_forbid_flag = self.DB_True
else:
message = message + '[业务元数据的质检中，没有项目{0}，请进行修正！]'.format(qa_name)
access_forbid_flag = self.DB_True
# 数据库部分
access_wait_flag, access_forbid_flag, message = \
self.db_access_check(access_wait_flag, access_forbid_flag, message)
# 开始进行检查的结果判断
access_flag = self.DataAccess_Pass
if access_forbid_flag:
access_flag = self.DataAccess_Forbid
elif access_wait_flag:
access_flag = self.DataAccess_Wait
if CUtils.equal_ignore_case(message, ''):
message = '模块可以进行访问!'
result = CResult.merge_result(
self.Success,
'模块[{0}.{1}]对对象[{2}]的访问能力已经分析完毕!分析结果为:{3}'.format(
CUtils.dict_value_by_name(self.information(), self.Name_ID, ''),
CUtils.dict_value_by_name(self.information(), self.Name_Title, ''),
self._obj_name,
message
)
)
return CResult.merge_result_info(result, self.Name_Access, access_flag)
except Exception as error:
result = CResult.merge_result(
self.Failure,
'模块[{0}.{1}]对对象[{2}]的访问能力的分析存在异常!详细情况: {3}!'.format(
CUtils.dict_value_by_name(self.information(), self.Name_ID, ''),
CUtils.dict_value_by_name(self.information(), self.Name_Title, ''),
self._obj_name,
error.__str__()
)
)
return CResult.merge_result_info(result, self.Name_Access, self.DataAccess_Forbid)
