def __a_check_file__(cls, result_template: dict, file_name_with_path: str, qa_items: dict) -> list: """ 根据规则, 验证文件的合法性 完成 负责人 赵宇飞 在这里对文件的其他内容进行质检, 目前实现了检查文件大小, 请参考__a_check_file_size__完善其他内容, 包括并不限于 验证文件可读性, 验证元数据文件可读性, 验证元数据文件的格式(xml\json), 以便于后面在处理元数据时, 不会出现异常 :param result_template 检查结果的模板 :param file_name_with_path: 文件名 :param qa_items: 检查项目, keywords :return: """ result_list = list() result_list.append( cls.__a_check_file_size__( result_template, file_name_with_path, CJson.dict_attr_by_path( qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Min), -1), CJson.dict_attr_by_path( qa_items, '{0}.{1}'.format(cls.Name_Size, cls.Name_Max), -1))) result_list.append( cls.__a_check_file_format__( result_template, file_name_with_path, CJson.dict_attr_by_path(qa_items, cls.Name_Format, None))) return result_list
def get_extent_by_vectorlayer(self, layer, feature_count) -> CJson: """ 构建图层四至范围的json对象 @param layer: @param feature_count: @return: """ json_extent = CJson() if feature_count == 0: json_extent.set_value_of_name("minx", 0) json_extent.set_value_of_name("maxx", 0) json_extent.set_value_of_name("miny", 0) json_extent.set_value_of_name("maxy", 0) else: extent = layer.GetExtent() # print('extent:', extent) if extent is not None: # print('ul:', extent[0], extent[3]) # print('lr:', extent[1], extent[2]) json_extent.set_value_of_name("minx", extent[0]) json_extent.set_value_of_name("maxx", extent[1]) json_extent.set_value_of_name("miny", extent[2]) json_extent.set_value_of_name("maxy", extent[3]) extent = None return json_extent
def create_view_json(self, params_json: CJson): """ 设置为一个参数,供进程调用 """ image_path = params_json.xpath_one('image_path', None) browse_full_path = params_json.xpath_one('browse_full_path', None) thumb_full_path = params_json.xpath_one('thumb_full_path', None) geotiff_full_path = params_json.xpath_one('geotiff_full_path', None) result = self.create_view(image_path, browse_full_path, thumb_full_path, geotiff_full_path) return result
def get_sync_dict_list(self, insert_or_updata) -> list: """ insert_or_updata 指明配置的是更新还是插入,-1时为插入,0为更新 本方法的写法为强规则,调用add_value_to_sync_dict_list配置 第一个参数为list,第二个参数为字段名,第三个参数为字段值,第四个参数为特殊配置 """ sync_dict_list = self.get_sync_predefined_dict_list(insert_or_updata) object_table_data = self._dataset # 时间信息 dso_time = object_table_data.value_by_name(0, 'dso_time', '') dso_time_json = CJson() dso_time_json.load_obj(dso_time) self.add_value_to_sync_dict_list( sync_dict_list, 'begdate', dso_time_json.xpath_one('start_time', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'enddate', dso_time_json.xpath_one('end_time', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'imagedate', CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', ''))) self.add_value_to_sync_dict_list( sync_dict_list, 'producetime', CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', ''))) self.add_value_to_sync_dict_list( sync_dict_list, 'imagedatetag', self.transform_time_to_imagedatetag( CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', '')))) if insert_or_updata: self.add_value_to_sync_dict_list(sync_dict_list, 'isdel', '1') return sync_dict_list
def process(self) -> str: """ 完成 负责人 张源博、赵宇飞 在这里提取影像数据的快视图, 将元数据文件存储在self.file_content.view_root_dir下 注意返回的串中有快视图和拇指图的文件名 注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!! :return: """ # 获取对象类型 type = 'default' group = 'default' catalog = 'default' # 构建数据对象object对应的识别插件,获取get_information里面的信息 class_classified_obj = CObject.get_plugins_instance_by_object_id(self.file_info.db_server_id, self.object_id) if class_classified_obj is not None: plugins_info = class_classified_obj.get_information() type = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Type, 'default') group = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Group, 'default') catalog = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Catalog, 'default') create_time = CTime.today() create_format_time = CTime.format_str(create_time, '%Y%m%d') year = CTime.format_str(create_time, '%Y') month = CTime.format_str(create_time, '%m') day = CTime.format_str(create_time, '%d') sep = CFile.sep() # 操作系统的不同处理分隔符不同 sep_list = [catalog, group, type, year, month, day] relative_path_part = sep.join(sep_list) # 相对路径格式 view_relative_path_browse = r'{2}{0}{2}{1}_browse.png'.format(relative_path_part, self.object_id, sep) view_relative_path_thumb = r'{2}{0}{2}{1}_thumb.jpg'.format(relative_path_part, self.object_id, sep) view_relative_path_geotiff = r'{2}{0}{2}{1}_browse.tiff'.format(relative_path_part, self.object_id, sep) browse_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_browse) thumb_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_thumb) geotiff_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_geotiff) # 进程调用模式 json_out_view = CJson() json_out_view.set_value_of_name('image_path', self.file_info.file_name_with_full_path) json_out_view.set_value_of_name('browse_full_path', browse_full_path) json_out_view.set_value_of_name('thumb_full_path', thumb_full_path) json_out_view.set_value_of_name('geotiff_full_path', geotiff_full_path) result_view = CProcessUtils.processing_method(self.create_view_json, json_out_view) # result_view = self.create_view(self.file_info.file_name_with_full_path, browse_full_path, thumb_full_path, # geotiff_full_path) # result_view = self.create_view_json(json_out_view) if CResult.result_success(result_view): result = CResult.merge_result(self.Success, '处理完毕!') result = CResult.merge_result_info(result, self.Name_Browse, view_relative_path_browse) result = CResult.merge_result_info(result, self.Name_Thumb, view_relative_path_thumb) result = CResult.merge_result_info(result, self.Name_Browse_GeoTiff, view_relative_path_geotiff) else: result = result_view return result
def merge_result(cls, result, message=None, base=None) -> str: new_result = CJson() if base is not None: new_result.load_json_text(base) new_result.set_value_of_name(cls.Name_Result, result) if message is not None: new_result.set_value_of_name(cls.Name_Message, message) return new_result.to_json()
def white_black_valid(self): """ 检查指定文件是否符合白名单, 黑名单验证 """ ds_storage_option = self._ds_storage.value_by_name( 0, 'dstotheroption', None) if ds_storage_option == '' or ds_storage_option is None: return True dir_filter_white_list = CJson.json_attr_value( ds_storage_option, self.Path_SO_Inbound_Filter_Dir_WhiteList, '') dir_filter_black_list = CJson.json_attr_value( ds_storage_option, self.Path_SO_Inbound_Filter_Dir_BlackList, '') file_filter_white_list = CJson.json_attr_value( ds_storage_option, self.Path_SO_Inbound_Filter_File_WhiteList, '') file_filter_black_list = CJson.json_attr_value( ds_storage_option, self.Path_SO_Inbound_Filter_File_BlackList, '') result = True if self.file_type != self.FileType_Unknown: if (dir_filter_white_list != '') and (dir_filter_black_list != ''): result = CFile.file_match( self.file_path_with_rel_path, dir_filter_white_list) and (not CFile.file_match( self.file_path_with_rel_path, dir_filter_black_list)) elif dir_filter_white_list != '': result = CFile.file_match(self.file_path_with_rel_path, dir_filter_white_list) elif dir_filter_black_list != '': result = not CFile.file_match(self.file_path_with_rel_path, dir_filter_black_list) if not result: return result if self.file_type == self.FileType_File: if (file_filter_white_list != '') and (file_filter_black_list != ''): return CFile.file_match( self.file_name_without_path, file_filter_white_list) and (not CFile.file_match( self.file_name_without_path, file_filter_black_list)) elif file_filter_white_list != '': return CFile.file_match(self.file_name_without_path, file_filter_white_list) elif file_filter_black_list != '': return not CFile.file_match(self.file_name_without_path, file_filter_black_list) else: return True else: return True
def process(self) -> str: """ """ browse_full_path = CFile.join_file(self.view_path, '{0}_browse.png'.format(self.object_id)) thumb_full_path = CFile.join_file(self.view_path, '{0}_thumb.jpg'.format(self.object_id)) geotiff_full_path = CFile.join_file(self.view_path, '{0}_browse.tiff'.format(self.object_id)) # 进程调用模式 json_out_view = CJson() json_out_view.set_value_of_name('image_path', self.transform_file) json_out_view.set_value_of_name('browse_full_path', browse_full_path) json_out_view.set_value_of_name('thumb_full_path', thumb_full_path) json_out_view.set_value_of_name('geotiff_full_path', geotiff_full_path) result_view = CProcessUtils.processing_method(self.create_view_json, json_out_view) # result_view = self.create_view(self.file_info.file_name_with_full_path, browse_full_path, thumb_full_path, # geotiff_full_path) # result_view = self.create_view_json(json_out_view) if CResult.result_success(result_view): # 清理不必要的文件 delect_file_list = list() delect_file_list.append('{0}.aux.xml'.format(browse_full_path)) delect_file_list.append('{0}.aux.xml'.format(thumb_full_path)) delect_file_list.append(geotiff_full_path) for delect_file in delect_file_list: if CFile.file_or_path_exist(delect_file): CFile.remove_file(delect_file) result = CResult.merge_result(self.Success, '处理完毕!') result = CResult.merge_result_info(result, self.Name_Browse, CFile.file_name(browse_full_path)) result = CResult.merge_result_info(result, self.Name_Thumb, CFile.file_name(thumb_full_path)) result = CResult.merge_result_info(result, self.Name_Browse_GeoTiff, CFile.file_name(geotiff_full_path)) else: # 清理不必要的文件 delect_file_list = list() delect_file_list.append(browse_full_path) delect_file_list.append(thumb_full_path) delect_file_list.append('{0}.aux.xml'.format(browse_full_path)) delect_file_list.append('{0}.aux.xml'.format(thumb_full_path)) delect_file_list.append(geotiff_full_path) for delect_file in delect_file_list: if CFile.file_or_path_exist(delect_file): CFile.remove_file(delect_file) result = result_view return result
def a_json_element(cls, audit_id, audit_title, audit_group, audit_result, json_obj: CJson, xpath: str, qa_items: dict) -> list: """ 判断一个json元数据中, 指定的jsonpath, 对应的element, 满足qa_items参数中的检测项目 :param audit_id: :param audit_title: :param audit_group: :param audit_result: :param json_obj: :param xpath: :param qa_items: :return: """ result_dict = cls.__init_audit_dict__(audit_id, audit_title, audit_group, audit_result) if json_obj is None: result_dict[cls.Name_Message] = 'Json对象不合法, 节点[{0}]不存在'.format( xpath) return [result_dict] json_value = json_obj.xpath_one(xpath, None) if json_value is not None: return cls.__a_check_value__(result_dict, json_value, '属性[{0}]'.format(audit_title), qa_items) else: result_dict[cls.Name_Message] = 'Json对象的节点[{0}]不存在, 请检查修正!'.format( xpath) return [result_dict]
def set_value_from_file(self, file_name: str, file_format: str, file_encoding: str): """ 设置value为文件内容 . 根据文件格式, 可以加载文件内容 . 如果文件格式为二进制, 则value存储文件名 :param file_name: :param file_format: :param file_encoding: :return: """ if CUtils.equal_ignore_case(file_format, CResource.FileFormat_TXT): self._value = { CResource.Name_Text: CFile.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_XML): self._value = { CResource.Name_Text: CXml.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } elif CUtils.equal_ignore_case(file_format, CResource.FileFormat_Json): self._value = { CResource.Name_Text: CJson.file_2_str(file_name), CResource.Name_Type: CResource.DataValueType_SQL } else: self._value = { CResource.Name_Text: file_name, CResource.Name_Type: CResource.DataValueType_File }
def test_get_attr(self): json = CJson() json.load_json_text(self.test_text) assert json.xpath_one('a', -1) == 1 assert json.xpath_one('aa', -1) == -1 assert json.xpath_one('student[0].name', '') == '小明' assert json.xpath_one('student[1].name', '') == '小王'
def set_metadata_bus(self, metadata_bus_extract_result: int, metadata_bus_extract_memo: str, metadata_bus_type: int, metadata_bus_text): self.__metadata_bus_extract_result__ = metadata_bus_extract_result self.__metadata_bus_extract_memo__ = metadata_bus_extract_memo self.__metadata_bus_type__ = metadata_bus_type if self.__metadata_bus_type__ == self.MetaDataFormat_Json: self.__metadata_bus_text__ = '' self.__metadata_bus_xml__ = CXml() self.__metadata_bus_json__.load_json_text(metadata_bus_text) elif self.__metadata_bus_type__ == self.MetaDataFormat_XML: self.__metadata_bus_text__ = '' self.__metadata_bus_xml__.load_xml(metadata_bus_text) self.__metadata_bus_json__ = CJson() else: self.__metadata_bus_text__ = metadata_bus_text self.__metadata_bus_xml__ = CXml() self.__metadata_bus_json__ = CJson()
def set_metadata_bus_file(self, metadata_bus_extract_result: int, metadata_bus_extract_memo: str, metadata_type: int, file_name): self.__metadata_bus_extract_result__ = metadata_bus_extract_result self.__metadata_bus_extract_memo__ = metadata_bus_extract_memo self.__metadata_bus_type__ = metadata_type if self.__metadata_bus_type__ == self.MetaDataFormat_Json: self.__metadata_bus_text__ = '' self.__metadata_bus_xml__ = CXml() self.__metadata_bus_json__.load_file(file_name) elif self.__metadata_bus_type__ == self.MetaDataFormat_XML: self.__metadata_bus_text__ = '' self.__metadata_bus_xml__.load_file(file_name) self.__metadata_bus_json__ = CJson() else: self.__metadata_bus_text__ = CFile.file_2_str(file_name) self.__metadata_bus_xml__ = CXml() self.__metadata_bus_json__ = CJson()
def params_value_by_name(self, attr_name: str, default_value): """ 通过解析传入参数, 直接获取任务执行方面的参数, 该参数都存储在trigger对象下 :param attr_name: :param default_value: :return: """ return CJson().json_attr_value( self.__schedule_params__, '{0}.{1}'.format(self.TRIGGER_Params, attr_name), default_value)
def params_value_by_name(self, params: str, attr_name: str, default_value): """ 通过解析传入参数, 直接获取任务执行方面的参数, 该参数都存储在job对象下 :param params: :param attr_name: :param default_value: :return: """ return CJson().json_attr_value( params, '{0}.{1}'.format(self.Name_Process, attr_name), default_value)
def register_dm_metadata_plugins(self): sql_register_dm_metadata_plugins_clear = ''' truncate table dm2_storage_object_def cascade ''' sql_unregister_dm_metadata_plugins = ''' delete from dm2_storage_object_def where dsodid = :dsodid ''' sql_register_dm_metadata_plugins = ''' insert into dm2_storage_object_def( dsodid, dsodtitle, dsodtype, dsodtypetitle, dsodtypecode, dsodgroup, dsodgrouptitle, dsodcatalog, dsodcatalogtitle, dsod_otheroption) values (:dsodid, :dsodtitle, :dsodtype, :dsodtypetitle, :dsodtypecode, :dsodgroup, :dsodgrouptitle, :dsodcatalog, :dsodcatalogtitle, :dsod_otheroption) ''' CFactory().give_me_db().execute(sql_register_dm_metadata_plugins_clear) plugins_root_dir = CSys.get_plugins_root_dir() plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir) for plugins_type in plugins_type_list: if CFile.is_dir(CFile.join_file( plugins_root_dir, plugins_type)) and ( not (str(plugins_type)).startswith('_')): plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), plugins_type) path = CFile.join_file(CSys.get_plugins_root_dir(), plugins_type) plugins_file_list = CFile.file_or_subpath_of_path( path, '{0}_*.{1}'.format(self.Name_Plugins, self.FileExt_Py)) for file_name_without_path in plugins_file_list: file_main_name = CFile.file_main_name( file_name_without_path) class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, file_main_name, None) plugins_info = class_classified_obj.get_information() json_obj = CJson() json_obj.set_value_of_name( self.Name_Is_Spatial, CUtils.dict_value_by_name( plugins_info, CPlugins.Plugins_Info_Is_Spatial, False)) json_obj.set_value_of_name( self.Name_Is_DataSet, CUtils.dict_value_by_name( plugins_info, CPlugins.Plugins_Info_Is_Dataset, False)) plugins_info['dsod_otheroption'] = json_obj.to_json() print('{0}/{1}:{2}'.format(plugins_type, file_main_name, plugins_info)) CFactory().give_me_db().execute( sql_unregister_dm_metadata_plugins, plugins_info) CFactory().give_me_db().execute( sql_register_dm_metadata_plugins, plugins_info)
def __init__(self): self.__quality__ = CQuality() self.__metadata_extract_result__ = self.Not_Support self.__metadata_extract_memo__ = '' self.__metadata_text__ = None self.__metadata_xml__ = CXml() self.__metadata_json__ = CJson() self.__metadata_type__ = self.MetaDataFormat_Text self.__metadata_bus_extract_result__ = self.Not_Support self.__metadata_bus_extract_memo__ = '' self.__metadata_bus_text__ = None self.__metadata_bus_xml__ = CXml() self.__metadata_bus_json__ = CJson() self.__metadata_bus_type__ = self.MetaDataFormat_Text self.__thumb_img_file_name__ = '' self.__browse_img_file_name__ = '' self.__metadata_view_extract_result__ = self.Not_Support self.__metadata_view_extract_memo__ = '' self.__time_information__ = CJson() self.__metadata_time_extract_result__ = self.Not_Support self.__metadata_time_extract_memo__ = '' self.__metadata_spatial_extract_result__ = self.Not_Support self.__metadata_spatial_extract_memo__ = '' self.__metadata_spatial__ = CMDSpatial()
def summary(self) -> str: json_obj = CJson() json_obj.set_value_of_name( self.Name_Total, self.__quality_result_of_level(self.__XPath_Total)) metadata_qa_s = { self.Name_Data: self.__quality_result_of_level(self.__XPath_MetaData_Data), self.Name_Business: self.__quality_result_of_level(self.__XPath_MetaData_Bus) } json_obj.set_value_of_name(self.Name_MetaData, metadata_qa_s) json_obj.set_value_of_name( self.Name_Data, { self.Name_Items: self.__quality_result_of_level(self.__XPath_Data_Items) }) return json_obj.to_json()
def params_value_by_name(self, attr_name: str, default_value): """ 通过解析传入参数, 直接获取任务执行方面的参数, 该参数都存储在job对象下 :param attr_name: :param default_value: :return: """ if self.__params is None: return default_value else: return CJson.json_attr_value( self.__params, '{0}.{1}'.format(self.NAME_JOB, attr_name), default_value)
def process(self) -> str: """ 在这里提取矢量数据的元数据, 将元数据文件存储在self.file_content.work_root_dir下, 固定名称为self.FileName_MetaData, 注意返回的串中有元数据的格式 :return: """ result = super().process() ds_metadata = CFactory().give_me_db( self.file_info.db_server_id).one_row( ''' select dsometadatatext, dsometadatajson, dsometadataxml from dm2_storage_object where dsoid = :object_id ''', {'object_id': self.object_id}) metadata_filename = CFile.join_file( self.file_content.work_root_dir, '{0}.metadata'.format(self.object_name)) text_metadata = ds_metadata.value_by_name(0, 'dsometadatatext', None) json_metadata = ds_metadata.value_by_name(0, 'dsometadatajson', None) xml_metadata = ds_metadata.value_by_name(0, 'dsometadataxml', None) format_metadata = self.MetaDataFormat_Text if json_metadata is not None: format_metadata = self.MetaDataFormat_Json CJson.str_2_file(json_metadata, metadata_filename) elif xml_metadata is not None: format_metadata = self.MetaDataFormat_XML CXml.str_2_file(xml_metadata, metadata_filename) elif text_metadata is not None: format_metadata = self.MetaDataFormat_XML CFile.str_2_file(text_metadata, metadata_filename) else: return result result = CResult.merge_result_info(result, self.Name_FileName, metadata_filename) return CResult.merge_result_info(result, self.Name_Format, format_metadata)
def get_other_metadata_by_raster(self, other_metadata: dict) -> CJson: """ 获取栅格文件的subdatasets、geolocation、rpc元数据信息 :param other_metadata: :return: """ other_data = CJson() other_data.set_value_of_name('valid', True) other_metadata_list = [] for i in other_metadata: if other_metadata[i] is not None: sub_metadata_item = other_metadata[i] other_metadata_list.append(sub_metadata_item) other_data.set_value_of_name('metadata', other_metadata_list) return other_data
def get_geometry_by_vectorlayer(self, layer) -> CJson: """ 构建图层图形类型的json对象 @param layer: @return: """ json_geometry = CJson() geomtype = layer.GetGeomType() # 数字 json_geometry.set_value_of_name("type", geomtype) feature_count = layer.GetFeatureCount() if feature_count > 0: for i in range(feature_count): feature = layer.GetFeature(i) if feature is not None: geom = feature.geometry() if geom is not None: geom_name = geom.GetGeometryName() json_geometry.set_value_of_name("name", geom_name) return json_geometry return json_geometry
def get_sync_predefined_dict_list(self, insert_or_updata) -> list: """ insert_or_updata 指明配置的是更新还是插入,-1时为插入,0为更新 本方法的写法为强规则,调用add_value_to_sync_dict_list配置 第一个参数为list,第二个参数为字段名,第三个参数为字段值,第四个参数为特殊配置 本方法处理公共部分 datacount:数据量 secrecylevel:密级 regioncode:行政区码 regionname:行政区 resolution:分辨率 colormodel:色彩模式 piexldepth:像素位数 scale:比例尺分母 mainrssource:主要星源 交插件去处理 """ sync_dict_list = list() object_table_id = self._obj_id # 获取oid object_table_data = self._dataset self.add_value_to_sync_dict_list(sync_dict_list, 'aprid', object_table_id) self.add_value_to_sync_dict_list( sync_dict_list, 'productname', object_table_data.value_by_name(0, 'dsoobjectname', '')) self.add_value_to_sync_dict_list(sync_dict_list, 'producttype', self._obj_type_code) self.add_value_to_sync_dict_list( sync_dict_list, 'dsodatatype', object_table_data.value_by_name(0, 'dsodatatype', '')) dso_time = object_table_data.value_by_name(0, 'dso_time', '') dso_time_json = CJson() dso_time_json.load_obj(dso_time) self.add_value_to_sync_dict_list( sync_dict_list, 'begdate', dso_time_json.xpath_one('start_time', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'enddate', dso_time_json.xpath_one('end_time', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'imagedate', CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', ''))) # datacount:数据数量 # secrecylevel:密级 # regioncode:行政区码 # regionname:行政区 上面四个字段交插件处理 self.add_value_to_sync_dict_list( # 配置子查询,调用函数 sync_dict_list, 'centerx', ''' (select st_x(st_centroid( (select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}') ))::decimal(8, 2)) '''.format(object_table_id), self.DataValueType_SQL) self.add_value_to_sync_dict_list( sync_dict_list, 'centery', ''' (select st_y(st_centroid( (select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}') ))::decimal(8, 2)) '''.format(object_table_id), self.DataValueType_SQL) self.add_value_to_sync_dict_list( sync_dict_list, 'geomwkt', ''' st_astext( (select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}') ) '''.format(object_table_id), self.DataValueType_SQL) self.add_value_to_sync_dict_list( sync_dict_list, 'geomobj', ''' (select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}') '''.format(object_table_id), self.DataValueType_SQL) self.add_value_to_sync_dict_list( sync_dict_list, 'browserimg', object_table_data.value_by_name(0, 'dso_browser', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'thumbimg', object_table_data.value_by_name(0, 'dso_thumb', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'producetime', CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', ''))) # resolution:分辨率,交插件处理 self.add_value_to_sync_dict_list( sync_dict_list, 'imgsize', ''' (select round((sum(dodfilesize)/1048576),2) from dm2_storage_obj_detail where dodobjectid='{0}') '''.format(object_table_id), self.DataValueType_SQL) # colormodel:交插件处理 # piexldepth:交插件处理 if insert_or_updata: self.add_value_to_sync_dict_list(sync_dict_list, 'isdel', '0') now_time = CUtils.any_2_str( datetime.datetime.now().strftime('%F %T')) self.add_value_to_sync_dict_list(sync_dict_list, 'addtime', now_time) self.add_value_to_sync_dict_list( sync_dict_list, 'extent', "(select dso_geo_bb_wgs84 from dm2_storage_object where dsoid='{0}')" .format(object_table_id), self.DataValueType_SQL) self.add_value_to_sync_dict_list( sync_dict_list, 'proj', object_table_data.value_by_name(0, 'dso_prj_wkt', '')) # remark:暂时为空 # ispublishservice:暂时为空 self.add_value_to_sync_dict_list(sync_dict_list, 'queryable', '0') # scale:交插件处理 # mainrssource:交插件处理 self.add_value_to_sync_dict_list( sync_dict_list, 'dsdid', object_table_data.value_by_name(0, 'query_directory_id', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'dsfid', object_table_data.value_by_name(0, 'query_file_id', '')) self.add_value_to_sync_dict_list( sync_dict_list, 'imagedatetag', self.transform_time_to_imagedatetag( CUtils.to_day_format(dso_time_json.xpath_one('time', ''), dso_time_json.xpath_one('time', '')))) return sync_dict_list
def execute(self) -> str: inbound_ib_n_list = CFactory().give_me_db( self.get_mission_db_id()).all_row(''' select dsiid as query_ib_id , dsiotheroption as query_ib_option , dsidirectory as query_ib_relation_dir , dsidirectoryid as query_ib_dir_id , dsitargetstorageid as query_ib_target_storage_id from dm2_storage_inbound where dsi_na_status = {0} '''.format(self.ProcStatus_WaitConfirm)) if inbound_ib_n_list.is_empty(): return CResult.merge_result(CResult.Success, '本次没有需要检查的通知任务!') for data_index in range(inbound_ib_n_list.size()): ds_ib_id = inbound_ib_n_list.value_by_name(data_index, 'query_ib_id', '') ds_ib_option = CUtils.any_2_str( inbound_ib_n_list.value_by_name(data_index, 'query_ib_option', '')) ds_ib_directory_name = inbound_ib_n_list.value_by_name( data_index, 'query_ib_relation_dir', '') ds_ib_directory_id = inbound_ib_n_list.value_by_name( data_index, 'query_ib_dir_id', '') ds_ib_target_storage_id = inbound_ib_n_list.value_by_name( data_index, 'query_ib_target_storage_id', '') module_name_list = CJson.json_attr_value( ds_ib_option, self.Path_IB_Opt_Notify_module, None) if module_name_list is None: modules_root_dir = CSys.get_metadata_data_access_modules_root_dir( ) module_file_list = CFile.file_or_subpath_of_path( modules_root_dir, '{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py)) module_name_list = list() for module_file in module_file_list: module_name_list.append(CFile.file_main_name(module_file)) CLogger().debug('正在检查入库批次[ds_ib_id]的通知进度...'.format(ds_ib_id)) try: # 所有通知对象的统计数 sql_record_total_count = CUtils.replace_placeholder( ''' select count(*) from dm2_storage_obj_na where dson_app_id in ($module_name_list) and dson_object_id in ( select dsoid from dm2_storage_object where dso_ib_id = :ib_id ) ''', { 'module_name_list': CUtils.list_2_str(module_name_list, "'", ',', "'") }) record_total_count = CFactory().give_me_db( self.get_mission_db_id()).one_value( sql_record_total_count, {'ib_id': ds_ib_id}, 0) if record_total_count == 0: self.update_inbound_na_result( ds_ib_id, CResult.merge_result( self.Failure, '入库任务下没有可通知的数据, 请检查异常情况! '.format(ds_ib_id))) continue # 已经完成的通知对象的统计数, 包括正常完成和错误的 sql_record_finished_count = CUtils.replace_placeholder( ''' select count(*) from dm2_storage_obj_na where dson_notify_status in ({0}, {1}) and dson_app_id in ($module_name_list) and dson_object_id in ( select dsoid from dm2_storage_object where dso_ib_id = :ib_id ) '''.format(self.ProcStatus_Finished, self.ProcStatus_Error), { 'module_name_list': CUtils.list_2_str(module_name_list, "'", ',', "'") }) record_finished_count = CFactory().give_me_db( self.get_mission_db_id()).one_value( sql_record_finished_count, {'ib_id': ds_ib_id}, 0) # 错误的记录数 sql_record_error_count = CUtils.replace_placeholder( ''' select count(*) from dm2_storage_obj_na where dson_notify_status = {0} and dson_app_id in ($module_name_list) and dson_object_id in ( select dsoid from dm2_storage_object where dso_ib_id = :ib_id ) '''.format(self.ProcStatus_Error), { 'module_name_list': CUtils.list_2_str(module_name_list, "'", ',', "'") }) record_error_count = CFactory().give_me_db( self.get_mission_db_id()).one_value( sql_record_error_count, {'ib_id': ds_ib_id}, 0) if record_total_count != record_finished_count: message = '入库任务[{0}]下的数据正在通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个...'.format( ds_ib_id, record_total_count, record_finished_count, record_error_count) CLogger().debug(message) self.update_inbound_na_progress( ds_ib_id, CResult.merge_result(self.Failure, message)) else: message = '入库任务[{0}]下的数据已经通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个, 请检查修正! '.format( ds_ib_id, record_total_count, record_finished_count, record_error_count) CLogger().debug(message) self.update_inbound_na_result( ds_ib_id, CResult.merge_result(self.Success, message)) except Exception as error: self.update_inbound_na_result( ds_ib_id, CResult.merge_result( self.Failure, '入库任务下的数据通知其他子系统过程中出现异常情况, 详细错误信息为: [{1}]'.format( ds_ib_id, error.__str__()))) continue return CResult.merge_result(self.Success, '本次通知监控任务成功结束!')
def to_file(cls, result_text, file_name): return CJson.str_2_file(result_text, file_name)
def result_message(cls, result_text) -> str: return CJson.json_attr_value(result_text, cls.Name_Message, '')
def result_info(cls, result_text, info_name: str, default_value): return CJson.json_attr_value(result_text, info_name, default_value)
def result_success(cls, result_text) -> bool: return CJson.json_attr_value(result_text, cls.Name_Result, cls.Failure) == cls.Success
def merge_result_info(cls, result_text, info_name: str, value): return CJson.json_set_attr(result_text, info_name, value)
def _do_access(self) -> str: try: quality_info_xml = self._quality_info # 获取质检xml quality_summary = self._dataset.value_by_name(0, 'dso_quality_summary', '') quality_summary_json = CJson() quality_summary_json.load_obj(quality_summary) access_wait_flag = self.DB_False # 定义等待标志,为True则存在检查项目为等待 access_forbid_flag = self.DB_False # 定义禁止标志,为True则存在检查项目为禁止 message = '' # 文件与影像质检部分 file_qa = quality_summary_json.xpath_one('total', '') image_qa = quality_summary_json.xpath_one('metadata.data', '') if CUtils.equal_ignore_case(file_qa, self.QA_Result_Error) \ or CUtils.equal_ignore_case(image_qa, self.QA_Result_Error): message = message + '[数据与其相关文件的质检存在error!请进行修正!]' access_forbid_flag = self.DB_True elif CUtils.equal_ignore_case(file_qa, self.QA_Result_Warn) \ or CUtils.equal_ignore_case(image_qa, self.QA_Result_Warn): message = message + '[数据与其相关文件的质检存在warn!请进行检查!]' access_wait_flag = self.DB_True elif CUtils.equal_ignore_case(quality_summary, ''): message = message + '[数据质检未进行,可能数据存在问题!请进行检查!]' access_forbid_flag = self.DB_True else: pass for qa_name, qa_id in self.access_check_dict().items(): # 循环写好的检查列表 # qa_id = CUtils.dict_value_by_name(access_check_dict, 'qa_id', '') # 获取id qa_node = quality_info_xml.xpath_one("//item[@id='{0}']".format(qa_id)) # 查询xml中的节点 if qa_node is not None: node_result = CXml.get_attr(qa_node, self.Name_Result, '', False) # 获取质检结果 if CUtils.equal_ignore_case(node_result, self.QA_Result_Pass): pass elif CUtils.equal_ignore_case(node_result, self.QA_Result_Warn): # 警告则等待 message = message + '[业务元数据的质检中,项目{0}不符合要求,建议修正!]'.format(qa_name) access_wait_flag = self.DB_True else: # 错误以及其他情况,比如'',或者为其他字段 message = message + '[业务元数据的质检中,项目{0}不符合要求,必须修改后方可入库!]'.format(qa_name) access_forbid_flag = self.DB_True else: message = message + '[业务元数据的质检中,没有项目{0},请进行修正!]'.format(qa_name) access_forbid_flag = self.DB_True # 数据库部分 access_wait_flag, access_forbid_flag, message = \ self.db_access_check(access_wait_flag, access_forbid_flag, message) # 开始进行检查的结果判断 access_flag = self.DataAccess_Pass if access_forbid_flag: access_flag = self.DataAccess_Forbid elif access_wait_flag: access_flag = self.DataAccess_Wait if CUtils.equal_ignore_case(message, ''): message = '模块可以进行访问!' result = CResult.merge_result( self.Success, '模块[{0}.{1}]对对象[{2}]的访问能力已经分析完毕!分析结果为:{3}'.format( CUtils.dict_value_by_name(self.information(), self.Name_ID, ''), CUtils.dict_value_by_name(self.information(), self.Name_Title, ''), self._obj_name, message ) ) return CResult.merge_result_info(result, self.Name_Access, access_flag) except Exception as error: result = CResult.merge_result( self.Failure, '模块[{0}.{1}]对对象[{2}]的访问能力的分析存在异常!详细情况: {3}!'.format( CUtils.dict_value_by_name(self.information(), self.Name_ID, ''), CUtils.dict_value_by_name(self.information(), self.Name_Title, ''), self._obj_name, error.__str__() ) ) return CResult.merge_result_info(result, self.Name_Access, self.DataAccess_Forbid)