def __prepare_where_condition(self, column: CColumn, sql_text: str, sql_params: dict): column_type = self.__database.db_column_type_by_name( column.db_column_type) if CUtils.equal_ignore_case(column_type.set_value_method, self.DB_Column_Set_Method_Param): sql_text = CUtils.str_append( sql_text, '{0}=:{1}'.format(column.name, self.__param_name(column.name)), ' and ') sql_params[self.__param_name(column.name)] = CUtils.any_2_str( CUtils.dict_value_by_name(column.value, self.Name_Text, '')) elif CUtils.equal_ignore_case(column_type.set_value_method, self.DB_Column_Set_Method_Function): sql_text = CUtils.str_append( sql_text, '{0}=:{1}'.format(column.name, self.__param_name(column.name)), ' and ') sql_params[self.__param_name( column.name)] = CUtils.replace_placeholder( column_type.set_value_template, dict({ self.Name_Value, CUtils.any_2_str( CUtils.dict_value_by_name(column.value, self.Name_Text, '')) })) return sql_text, sql_params
def txt_to_xml(self, file_metadata_name_with_path: str): """ 完成 王学谦 txt文件转xml,在函数外提前定义xml对象并获取父节点传入,函数会将通过父节点构造xml对象 by王学谦 :param file_metadata_name_with_path:查询的mat文件全名,带路径 :return xml_obj:将文件内容存储好的项目对象 """ text_list = CFile.file_2_list( file_metadata_name_with_path) # 获取mat文件作为列表 if (text_list is None) or len(text_list) == 0: raise Exception('元数据文件无法读取,请检查') # 如果获取的文件内容为空,则抛出异常 xml_obj = CXml() # 建立xml对象 node_root = xml_obj.new_xml('root') xml_obj.set_attr(node_root, self.Name_Type, self.Transformer_TXT) # 设置root节点与属性 for row_text in text_list: if not CUtils.equal_ignore_case(row_text, ''): row_list = re.split(r'=', row_text.strip()) # 利用正则表达式,根据一个或多个tab剪切字符 # item节点 node_item = xml_obj.create_element(node_root, 'item') xml_obj.set_attr(node_item, self.Name_Name, CUtils.any_2_str(row_list[0]).strip()) if len(row_list) > 1: item_value = CUtils.any_2_str(row_list[1].strip()) if item_value.startswith('"') and item_value.endswith('"'): item_value = item_value[1:-1] xml_obj.set_element_text(node_item, item_value) # 设置item节点与属性与内容 return xml_obj
def xls_to_xml(self, file_metadata_name_with_path: str): """ 完成 王学谦 xls/xlsx文件转xml,在函数外提前定义xml对象并获取父节点传入,函数会将通过父节点构造xml对象 by王学谦 :param file_metadata_name_with_path:查询的xls/xlsx文件全名,带路径 :return xml_obj:将文件内容存储好的项目对象 """ all_data = xlrd.open_workbook(file_metadata_name_with_path) # 获取表格所有内容 table_data = all_data.sheets()[0] # 默认获取第一个表格 cols_num = table_data.ncols # 获取列数 rows_num = table_data.nrows # 获取行数 cols_index = 0 # 预定义列的index if CUtils.equal_ignore_case(CUtils.any_2_str(cols_num), CUtils.any_2_str(2)): pass # 无序号列从1列开始 elif CUtils.equal_ignore_case(CUtils.any_2_str(cols_num), CUtils.any_2_str(3)): cols_index = 1 # 有序号列从2列开始 else: raise Exception('xls格式异常,无法正常解析') xml_obj = CXml() # 建立xml对象 node_root = xml_obj.new_xml('root') xml_obj.set_attr(node_root, self.Name_Type, self.transformer_type) # 设置root节点与属性 for row in range(0, rows_num): node_item = xml_obj.create_element(node_root, 'item') xml_obj.set_attr( node_item, self.Name_Name, CUtils.any_2_str(table_data.cell(row, cols_index).value).lower()) xml_obj.set_element_text( node_item, table_data.cell(row, cols_index + 1).value) # 设置item节点与属性与内容 return xml_obj
def mat_to_xml(self, file_metadata_name_with_path: str): """ 完成 王学谦 mat文件转xml,在函数外提前定义xml对象并获取父节点传入,函数会将通过父节点构造xml对象 by王学谦 :param file_metadata_name_with_path:查询的mat文件全名,带路径 :return xml_obj:将文件内容存储好的项目对象 """ text_list = CFile.file_2_list( file_metadata_name_with_path) # 获取mat文件作为列表 if (text_list is None) or len(text_list) == 0 or CUtils.equal_ignore_case( CUtils.any_2_str(text_list), ''): raise # 如果获取的文件内容为空,则抛出异常 flag = False # 设置标志 xml_obj = CXml() # 建立xml对象 node_root = xml_obj.new_xml('root') xml_obj.set_attr(node_root, self.Name_Type, self.transformer_type) # 设置root节点与属性 for index, row_text in enumerate(text_list): if row_text.startswith('1\t'): # 从开头为1+tab键的行开始录入 flag = True row_list = re.split(r'\s+', row_text) # 利用正则表达式,根据一个或多个tab剪切字符 if flag: node_item = xml_obj.create_element(node_root, 'item') xml_obj.set_attr(node_item, self.Name_Name, CUtils.any_2_str(row_list[1]).lower()) xml_obj.set_element_text( node_item, CUtils.any_2_str(row_list[2].strip())) # 设置item节点与属性与内容 if not flag: raise Exception('文件内容异常,无法正常识别文件开头') # 如果未找到1+tab键开头,则抛出异常 return xml_obj
def join_file(cls, path: str, *paths: AnyStr) -> str: result = CUtils.any_2_str(path) for each_path in paths: real_file_name = CUtils.any_2_str(each_path) if real_file_name.startswith(r'/') or real_file_name.startswith('\\'): real_file_name = real_file_name[1:len(real_file_name)] result = '{0}{1}{2}'.format(result, cls.sep(), real_file_name) return result
def db_check_and_update(self, ib_id): """ 检查并更新dm2_storage_directory表中记录 :return: """ if not self.ds_file_or_path.is_empty(): # 如果记录已经存在 db_path_modify_time = self.ds_file_or_path.value_by_name( 0, 'dsddirlastmodifytime', '') if CUtils.equal_ignore_case( CUtils.any_2_str(db_path_modify_time), CUtils.any_2_str(self.file_modify_time)): CLogger().info( '目录[{0}]的最后修改时间, 和库中登记的没有变化, 子目录将被设置为忽略刷新! '.format( self.file_name_with_full_path)) CFactory().give_me_db(self.db_server_id).execute( ''' update dm2_storage_directory set dsdScanStatus = 0, dsdScanFileStatus = 0, dsd_directory_valid = -1, dsd_ib_id = :ib_id, dsdscanmemo = :message where dsdid = :dsdid ''', { 'dsdid': self.my_id, 'ib_id': ib_id, 'message': '目录[{0}]的最后修改时间, 和库中登记的没有变化, 子目录将被设置为忽略刷新! '.format( self.file_name_with_full_path) }) else: CLogger().info( '目录[{0}]的最后修改时间, 和库中登记的有变化, 子目录将被设置为重新刷新! '.format( self.file_name_with_full_path)) CFactory().give_me_db(self.db_server_id).execute( ''' update dm2_storage_directory set dsdScanStatus = 1, dsdScanFileStatus = 1, dsd_directory_valid = -1, dsd_ib_id = :ib_id, dsdscanmemo = :message where dsdid = :dsdid ''', { 'dsdid': self.my_id, 'ib_id': ib_id, 'message': '目录[{0}]的最后修改时间, 和库中登记的有变化, 子目录将被设置为重新刷新! '.format( self.file_name_with_full_path) }) else: CLogger().info('目录[{0}]未在库中登记, 系统将登记该记录! '.format( self.file_name_with_full_path)) self.__db_insert(ib_id)
def get_mission_seize_sql(self) -> str: if self.__application_id is None: return ''' update sch_center_mission set scmprocessid = '{0}', scmstatus = 2 where scmid = ( select scmid from sch_center_mission where scmstatus = 1 and ( scmcenterid in (select scid from sch_center where scserver is null) or scmcenterid is null ) limit 1 for update skip locked ) '''.format(self.SYSTEM_NAME_MISSION_ID) else: return ''' update sch_center_mission set scmprocessid = '{0}', scmstatus = 2 where scmid = ( select scmid from sch_center_mission where scmstatus = 1 and ( scmcenterid in (select scid from sch_center where scserver = '{1}') ) limit 1 for update skip locked ) '''.format(self.SYSTEM_NAME_MISSION_ID, CUtils.any_2_str(self.__application_id))
def give_me_extractor(cls, extract_type, object_id: str, object_name: str, file_info: CDMFilePathInfoEx, file_content: CVirtualContent): input_parser_type = CUtils.any_2_str(extract_type) if CUtils.equal_ignore_case(input_parser_type, cls.MetaDataEngine_Raster): return CMDExtractorRaster(object_id, object_name, file_info, file_content) elif CUtils.equal_ignore_case(input_parser_type, cls.MetaDataEngine_Vector): return CMDExtractorVector(object_id, object_name, file_info, file_content) elif CUtils.equal_ignore_case(input_parser_type, cls.MetaDataEngine_Document_Tika): return CMDExtractorDocument_Tika(object_id, object_name, file_info, file_content) elif CUtils.equal_ignore_case(input_parser_type, cls.MetaDataEngine_Document): return CMDExtractorDocument(object_id, object_name, file_info, file_content) elif CUtils.equal_ignore_case(input_parser_type, cls.MetaDataEngine_Spatial_Layer): return CMDExtractorSpatialLayer(object_id, object_name, file_info, file_content) elif CUtils.equal_ignore_case(input_parser_type, cls.MetaDataEngine_Attached_File): return CMDExtractorAttachedFile(object_id, object_name, file_info, file_content) else: return CParserCustom(object_id, object_name, file_info)
def object_uni_full_name_by_id(self, object_id: str): result = CFactory().give_me_db(self.db_server_id).one_value( ''' select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name from dm2_storage_object, dm2_storage_file, dm2_storage where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id and dm2_storage_file.dsfstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{0}' and dm2_storage_object.dsoid = :object_id union select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name from dm2_storage_object, dm2_storage_directory, dm2_storage where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id and dm2_storage_directory.dsdstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{1}' and dm2_storage_object.dsoid = :object_id '''.format(self.FileType_File, self.FileType_Dir) , {'object_id': object_id} ) if result is not None: return result ds_object_info = CFactory().give_me_db(self.db_server_id).one_row( ''' select dsoobjectname, dsoparentobjid from dm2_storage_object where dsoid = :object_id and dsodatatype not in ('file', 'dir') ''' ) if ds_object_info.is_empty(): return None object_name = ds_object_info.value_by_index(0, 0, '') object_parent_id = ds_object_info.value_by_index(0, 1, '') if CUtils.equal_ignore_case(object_parent_id, ''): return None result = CFactory().give_me_db(self.db_server_id).one_value( ''' select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name from dm2_storage_object, dm2_storage_file, dm2_storage where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id and dm2_storage_file.dsfstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{0}' and dm2_storage_object.dsoid = :object_id union select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name from dm2_storage_object, dm2_storage_directory, dm2_storage where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id and dm2_storage_directory.dsdstorageid = dm2_storage.dstid and dm2_storage_object.dsodatatype = '{1}' and dm2_storage_object.dsoid = :object_id '''.format(self.FileType_File, self.FileType_Dir) , {'object_id': object_parent_id} ) if result is not None: return CFile.join_file(CUtils.any_2_str(result), object_name) else: return None
def should_stop(self) -> bool: if self.__application_id is None: sql = ''' select scmid from sch_center_mission where (scmcommand <> '{0}' or scmstatus <> {1}) and ( (scmcenterid is null) or (scmcenterid in (select scid from sch_center where scserver is null)) ) limit 1 '''.format(self.CMD_SHUTDOWN, self.Status_Finish) else: sql = ''' select scmid from sch_center_mission where (scmcommand <> '{0}' or scmstatus <> {1}) and (scmcenterid in (select scid from sch_center where scserver = '{2}')) limit 1 '''.format(self.CMD_SHUTDOWN, self.Status_Finish, CUtils.any_2_str(self.__application_id)) try: factory = CFactory() db = factory.give_me_db() return not db.if_exists(sql) except: return True
def process_mission(self, dataset) -> str: ds_id = dataset.value_by_name(0, 'query_dir_id', '') ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') inbound_id = dataset.value_by_name(0, 'query_dir_ib_id', None) ds_subpath = dataset.value_by_name(0, 'query_subpath', '') ds_root_path = dataset.value_by_name(0, 'query_rootpath', '') ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str( dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times)) self.update_dir_status(ds_id, process_result, self.ProcStatus_Error) return process_result if ds_subpath == '': ds_subpath = ds_root_path else: ds_subpath = CFile.join_file(ds_root_path, ds_subpath) # 将所有子目录, 文件的可用性, 都改为未知!!!! self.init_file_or_subpath_valid_unknown(ds_id) try: sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and position((dsddirectory || '{0}') in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_storage_id, 'dsdDirectory': ds_subpath }) ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '') CLogger().debug('处理的目录为: {0}'.format(ds_subpath)) self.parser_file_or_subpath_of_path(dataset, ds_id, ds_subpath, ds_rule_content, inbound_id) result = CResult.merge_result( self.Success, '目录为[{0}]下的文件和子目录扫描处理成功!'.format(ds_subpath)) self.update_dir_status(ds_id, result) return result except Exception as err: result = CResult.merge_result( self.Failure, '目录为[{0}]下的文件和子目录扫描处理出现错误!错误原因为: {1}'.format( ds_subpath, err.__str__())) self.update_dir_status(ds_id, result) return result finally: self.exchange_file_or_subpath_valid_unknown2invalid(ds_id)
def __import_each_record(self, data_source: CDataSetSeqReader, data_target: CTable) -> str: data_target.column_list.reset() data_source_record = data_source.record_as_dict() for column_index in range(data_target.column_list.size()): column_obj = data_target.column_list.column_by_index(column_index) column_name = column_obj.name column_data_set_method = self.__find_column_data_set_method(column_obj, data_source) if column_data_set_method is None: continue column_value_set_type = CUtils.dict_value_by_name(column_data_set_method, self.Name_Type, self.Name_Common) if CUtils.equal_ignore_case(column_value_set_type, self.Name_SQL): column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '') column_value_template = self.replace_placeholder(column_value_template) column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record) column_obj.set_sql(column_value_template) elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Geometry): column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '') column_value_template = self.replace_placeholder(column_value_template) column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record) column_obj.set_geometry( column_value_template, CUtils.dict_value_by_name(column_data_set_method, self.Name_Srid, None) ) elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Array): column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, None) if isinstance(column_value_template, list): column_obj.set_array(column_value_template) else: column_value_template = CUtils.any_2_str(column_value_template) column_value_template = self.replace_placeholder(column_value_template) column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record) column_obj.set_array_str(column_value_template) elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Null): column_obj.set_null() elif CUtils.equal_ignore_case(column_value_set_type, self.Name_File): column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '') column_value_template = self.replace_placeholder(column_value_template) column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record) file_format = CUtils.dict_value_by_name(column_data_set_method, self.Name_Format, self.Name_Binary) file_format = self.replace_placeholder(file_format) file_format = CUtils.replace_placeholder(file_format, data_source.record_as_dict()) file_encoding = CUtils.dict_value_by_name(column_data_set_method, self.Name_Encoding, self.Encoding_UTF8) file_encoding = self.replace_placeholder(file_encoding) file_encoding = CUtils.replace_placeholder(file_encoding, data_source_record) column_obj.set_value_from_file(column_value_template, file_format, file_encoding) else: column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '') column_value_template = self.replace_placeholder(column_value_template) column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record) column_obj.set_value(column_value_template) data_target.save_data() return CResult.merge_result(self.Success, '数据入库成功!')
def package(output_relation_dir): """ 编译根目录下的包括子目录里的所有py文件成pyc文件到新的文件夹下 :param output_relation_dir: 需编译的目录 :return: """ output_relation_dir = CFile.unify(CUtils.any_2_str(output_relation_dir)) application_dir = CFile.file_path(CFile.file_abs_path(__file__)) output_dir = CFile.file_abs_path(CFile.join_file(application_dir, output_relation_dir)) for each_directory, dir_name_list, file_name_without_path_list in os.walk(application_dir): directory_source = each_directory directory_name = CFile.file_name(directory_source) directory_relation = CFile.file_relation_path(each_directory, application_dir) directory_target = CFile.join_file(output_dir, directory_relation) path_deploy_enable = deploy_match_pattern_list(directory_relation, 'path.white_list', True, True) path_deploy_enable = path_deploy_enable and deploy_match_pattern_list(directory_relation, 'path.black_list', False, True) if path_deploy_enable: directory_deploy_enable = deploy_match_pattern_list(directory_name, 'directory.white_list', True, True) directory_deploy_enable = directory_deploy_enable and deploy_match_pattern_list( directory_name, 'directory.black_list', False, True ) if directory_deploy_enable: for file_name_without_path in file_name_without_path_list: file_deploy_enable = deploy_match_pattern_list( file_name_without_path, 'file.white_list', True, True ) file_deploy_enable = file_deploy_enable and deploy_match_pattern_list( file_name_without_path, 'file.black_list', False, True ) file_name_with_path_source = CFile.join_file(directory_source, file_name_without_path) if file_deploy_enable: file_compile_enable = deploy_match_pattern_list( file_name_without_path, 'compile.file.white_list', True, False ) if file_compile_enable: file_compile_enable = deploy_match_pattern_list( file_name_without_path, 'compile.file.black_list', False, False ) file_name_without_path_target = CFile.change_file_ext(file_name_without_path, 'pyc') file_name_with_path_target = CFile.join_file(directory_target, file_name_without_path_target) CFile.check_and_create_directory_itself(directory_target) if file_compile_enable: py_compile.compile(file_name_with_path_source, cfile=file_name_with_path_target) print('{0}-compile-success'.format(file_name_with_path_source)) else: CFile.copy_file_to(file_name_with_path_source, directory_target) print('{0}-no_compile'.format(file_name_with_path_source))
def record_as_dict(self) -> dict: data_record = self._data.record(self.__record_index) result = dict() key_list = data_record.keys() for key in key_list: result[CUtils.any_2_str(key).lower()] = CUtils.dict_value_by_name( data_record, key, None, False) return result
def json_set_attr(cls, json_text, attr_name: str, attr_value) -> str: new_result = CJson() rt_json_text = CUtils.any_2_str(json_text) if not CUtils.equal_ignore_case(rt_json_text, ''): new_result.load_json_text(rt_json_text) new_result.set_value_of_name(attr_name, attr_value) return new_result.to_json()
def set_element_text(cls, element, text): """ 设置一个节点的文本 :param element: :param text: :return: """ if element is not None: element.text = etree.CDATA(CUtils.any_2_str(text))
def list_item_match(self, data_sample_list, keyword_value, fuzzy_matching): """ 根据给定列表和关键字和匹配选项, 对列表是否包含关键字进行完全匹配或模糊匹配 :param data_sample_list: :param keyword_value: :param fuzzy_matching: :return: """ keyword_text = CUtils.any_2_str(keyword_value).lower() if fuzzy_matching: for data_sample in data_sample_list: data_sample_text = CUtils.any_2_str(data_sample).lower() if data_sample_text.find(keyword_text) > -1: return True else: if CUtils.list_count(data_sample_list, keyword_value) > 0: return True return False
def set_metadata_time(self, result: int, memo: str, time_attr_name=None, time_attr_value=None): self.__metadata_time_extract_result__ = result self.__metadata_time_extract_memo__ = memo if time_attr_name is not None: self.__time_information__.set_value_of_name( CUtils.any_2_str(time_attr_name), time_attr_value)
def txt_to_xml(self, file_metadata_name_with_path: str): """ 完成 王学谦 txt文件转xml,在函数外提前定义xml对象并获取父节点传入,函数会将通过父节点构造xml对象 by王学谦 :param file_metadata_name_with_path:查询的mat文件全名,带路径 :return xml_obj:将文件内容存储好的项目对象 """ text_list = CFile.file_2_list(file_metadata_name_with_path) # 获取mat文件作为列表 if (text_list is None) or len(text_list) == 0: raise Exception('元数据文件无法读取,请检查') # 如果获取的文件内容为空,则抛出异常 xml_obj = CXml() # 建立xml对象 node_root = xml_obj.new_xml('root') xml_obj.set_attr(node_root, self.Name_Type, self.Transformer_TXT) # 设置root节点与属性 # 设置操作的节点 current_node = node_root for row_text in text_list: if CUtils.equal_ignore_case('row_text', 'END'): break # 分割字符 row_list = re.split(r'=', row_text.strip()) if len(row_list) >= 2: # 为GROUP建立新节点 if CUtils.equal_ignore_case(row_list[0], 'GROUP'): node_item = xml_obj.create_element(current_node, 'item') # 值设为属性 item_value = CUtils.any_2_str(row_list[1].strip()) if item_value.startswith('"') and item_value.endswith('"'): item_value = item_value[1:-1] xml_obj.set_attr(node_item, self.Name_Name, item_value) current_node = node_item elif CUtils.equal_ignore_case(row_list[0], 'END_GROUP'): current_node = xml_obj.node_xpath_one(current_node, '..') else: node_item = xml_obj.create_element(current_node, 'item') xml_obj.set_attr(node_item, self.Name_Name, row_list[0].strip()) item_value = CUtils.any_2_str(row_list[1].strip()) if item_value.startswith('"') and item_value.endswith('"'): item_value = item_value[1:-1] xml_obj.set_element_text(node_item, item_value) return xml_obj
def load_xml(self, xml_content: str): """ 通过给定的xml内容, 对xml对象进行初始化 :param xml_content: :return: """ if not CUtils.equal_ignore_case(CUtils.any_2_str(xml_content), ''): parser = etree.XMLParser(remove_blank_text=True, strip_cdata=False) self.__xml_root_node = etree.XML(xml_content.strip(), parser) self.__xml_tree = etree.ElementTree(self.__xml_root_node)
def set_attr(cls, element, attr_name, attr_value): """ 设置一个节点的属性 :param element: :param attr_name: :param attr_value: :return: """ if element is not None: element.set(attr_name, CUtils.any_2_str(attr_value))
def get_wkt_to_transform_coordinates(self, metadata_bus_dict, metadata_bus_xml, multiple_metadata_bus_filename_dict): list_result = metadata_bus_xml.xpath( '/product/imageAttributes/geographicInformation/geolocationGrid/imageTiePoint' ) list_num = len(list_result) temp_str = '' temp_frist_flag = True temp_frist_str = '' for num in range(1, list_num + 1): try: latitude_node = metadata_bus_xml.xpath_one( '/product/imageAttributes/geographicInformation/geolocationGrid' '/imageTiePoint[{0}]/geodeticCoordinate/latitude'.format( num)) latitude = metadata_bus_xml.get_element_text(latitude_node) longitude_node = metadata_bus_xml.xpath_one( '/product/imageAttributes/geographicInformation/geolocationGrid' '/imageTiePoint[{0}]/geodeticCoordinate/longitude'.format( num)) longitude = metadata_bus_xml.get_element_text(longitude_node) if not CUtils.equal_ignore_case( latitude, '') and not CUtils.equal_ignore_case( longitude, ''): temp_str = temp_str + '{0} {1},'.format( CUtils.any_2_str(CUtils.to_decimal(latitude, latitude)), CUtils.any_2_str( CUtils.to_decimal(longitude, longitude))) if temp_frist_flag: temp_frist_str = temp_str[:-1] temp_frist_flag = False except Exception as error: raise Exception('计算wkt时出错,原因为{0}'.format(error.__str__())) if CUtils.equal_ignore_case(temp_str, ''): return '' else: temp_str = '{0}{1}'.format(temp_str, temp_frist_str) wkt = 'POLYGON( ({0}) )'.format(temp_str) return wkt
def __init__(self, job_id: str, job_params: str): self.__id = job_id self.__params = job_params self.__mission_db_id = CUtils.any_2_str( self.params_value_by_name(self.Job_Params_DB_Server_ID, self.DB_Server_ID_Default)) self.__abnormal_job_retry_times = self.params_value_by_name( self.Job_Params_Abnormal_Job_Retry_Times, -1) if CUtils.equal_ignore_case(self.__abnormal_job_retry_times, -1): self.__abnormal_job_retry_times = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_Parser_MetaData_Retry_Times, self.Default_Abnormal_Job_Retry_Times) self.custom_init()
def give_me_parser(cls, parser_type, object_id: str, object_name: str, file_info: CDMFilePathInfoEx, file_custom_list: list): input_parser_type = CUtils.any_2_str(parser_type) if CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_Same_File_Main_Name): return CDetailParser_Same_File_Main_Name(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_File_Itself): return CDetailParser_File_Itself(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_Directory_Itself): return CDetailParser_Directory_Itself(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_File_Of_Same_Dir): return CDetailParser_File_Of_Same_Dir(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_All_File_Of_Same_Dir): return CDetailParser_All_File_Of_Same_Dir(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_File_Of_Dir): return CDetailParser_File_Of_Dir(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_All_File_Of_Dir): return CDetailParser_All_File_Of_Dir(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case(input_parser_type, cls.DetailEngine_Fuzzy_File_Main_Name): return CDetailParser_Fuzzy_File_Main_Name(object_id, object_name, file_info, file_custom_list) elif CUtils.equal_ignore_case( input_parser_type, cls.DetailEngine_Busdataset): # 用于入数据集的附属文件metadata.21at return CDetailParser_Busdataset(object_id, object_name, file_info, file_custom_list) else: # 注意, 这里改为基类了, 因为基类中将默认的处理清除已有附属文件的逻辑 return CDetailParser(object_id, object_name, file_info, file_custom_list)
def process_metadata_table(self): object_table_id = self._obj_id # 获取oid object_table_data = self._dataset metadata_bus_dict = self.get_metadata_bus_dict() metadata_table_name = CUtils.dict_value_by_name( self.information(), 'metadata_table_name', 'ap_product_metadata') metadata_table = CTable() metadata_table.load_info(self._db_id, metadata_table_name) metadata_table.column_list.column_by_name('id').set_value( object_table_id) metadata_table.column_list.column_by_name('fid').set_value( object_table_id) dsometadataxml = object_table_data.value_by_name( 0, 'dsometadataxml_bus', None) metadata_table.column_list.column_by_name('metaxml').set_value( dsometadataxml) # otherxml = CUtils.dict_value_by_name(metadata_bus_dict, 'otherxml', None) # if not CUtils.equal_ignore_case(otherxml, ''): # view_path = settings.application.xpath_one(self.Path_Setting_MetaData_Dir_View, None) # browser_path = CFile.file_path(self._dataset.value_by_name(0, 'dso_browser', None)) # file_list = CFile.file_or_dir_fullname_of_path( # CFile.join_file(view_path, browser_path), False, otherxml, CFile.MatchType_Regex # ) # if len(file_list) > 0: # metadata_table.column_list.column_by_name('otherxml').set_value( # CFile.file_2_str(file_list[0]) # ) metadata_table.column_list.column_by_name('otherxml').set_value( '<?xml version="1.0" encoding="UTF-8"?><root><plugin name="{0}"/></root>' .format( CUtils.dict_value_by_name(metadata_bus_dict, 'satelliteid', None))) if not metadata_table.if_exists(): metadata_table.column_list.column_by_name('version').set_value( '1.0') now_time = CUtils.any_2_str( datetime.datetime.now().strftime('%F %T')) metadata_table.column_list.column_by_name('addtime').set_value( now_time) result = metadata_table.save_data() return result
def mdb_to_xml(self, file_metadata_name_with_path: str): """ TODO 王学谦 mdb文件转xml,在函数外提前定义xml对象并获取父节点传入,函数会将通过父节点构造xml对象 :param file_metadata_name_with_path:查询的mdb文件全名,带路径 :return xml_obj:将文件内容存储好的项目对象 """ conn = None # 预定义连接与游标,方便释放 cur = None try: conn = self.get_mdb_connect(file_metadata_name_with_path) cur = conn.cursor() # 游标 xml_obj = CXml() # 建立xml对象 node_root = xml_obj.new_xml('root') xml_obj.set_attr(node_root, self.Name_Type, self.transformer_type) # 设置root节点与属性 table_name_list = ['mbii', 'mpid', 'mppi', 'mqc1', 'mqc2', 'mdac'] for table_name in table_name_list: try: sql = "SELECT * FROM " + table_name cur.execute(sql) table_data = cur.fetchall() # total_rows = len(alldata) # 行 # total_cols = len(alldata[0]) # 列 node_property = xml_obj.create_element(node_root, 'property') xml_obj.set_attr(node_property, 'tablename', table_name) # 设置property节点与属性与内容 for field_index, row_obj in enumerate(cur.description): row_name = row_obj[0] # 字段名称 row_type = row_obj[1] # 字段类型 if row_type is bytearray: # 跳过长二进制数组 continue node_item = xml_obj.create_element(node_property, 'item') xml_obj.set_attr(node_item, self.Name_Name, CUtils.any_2_str(row_name).lower()) xml_obj.set_element_text(node_item, table_data[0][field_index]) # 设置item节点与属性与内容 except: continue except Exception as error: raise Exception(error.__str__()) finally: if cur is not None: cur.close() if conn is not None: conn.close() return xml_obj
def give_me_db(self, db_id=None) -> CDataBase: if db_id is None: rt_db_id = self.DB_Server_ID_Default else: rt_db_id = CUtils.any_2_str(db_id) if CUtils.equal_ignore_case(rt_db_id, ''): rt_db_id = self.DB_Server_ID_Default databases = settings.application.xpath_one(self.Name_DataBases, None) if databases is None: raise Exception('系统未配置数据库定义, 请检查修正后重试!') for database in databases: if rt_db_id == CUtils.dict_value_by_name(database, self.Name_ID, self.DB_Server_ID_Default): return self.create_db(database) raise Exception('未找到标示为[{0}]]的数据库定义!'.format(rt_db_id))
def get_abnormal_mission_restart_sql(self) -> str: if self.__application_id is None: return ''' update sch_center_mission set scmstatus = 1, scmprocessid = null where scmstatus = 2 and ( (scmcenterid is null) or (scmcenterid in (select scid from sch_center where scserver is null)) ) ''' else: return ''' update sch_center_mission set scmstatus = 1, scmprocessid = null where scmstatus = 2 and (scmcenterid in (select scid from sch_center where scserver = '{0}')) '''.format(CUtils.any_2_str(self.__application_id))
def seq_next_value(self, seq_type: int) -> str: if seq_type == self.Seq_Type_Date_AutoInc: sql_last_seq_date = ''' select coalesce(gcfgvalue, current_date::text) as last_date, current_date::text as curr_date from ro_global_config where gcfgcode = 'sys_seq_date_autoinc' ''' ds_last_seq_date = self.one_row(sql_last_seq_date) if ds_last_seq_date.is_empty(): self.execute(''' insert into ro_global_config(gcfgid, gcfgcode, gcfgtitle, gcfgvalue, gcfgmemo) values (10001, 'sys_seq_date_autoinc', '日期自增序列最后记录', current_date::text, null) ''') ds_last_seq_date = self.one_row(sql_last_seq_date) last_seq_date = ds_last_seq_date.value_by_name( 0, 'last_date', None) curr_seq_date = ds_last_seq_date.value_by_name( 0, 'curr_date', None) if last_seq_date is None: self.execute(''' update ro_global_config set gcfgvalue = current_date::text where gcfgcode = 'sys_seq_date_autoinc' ''') elif not CUtils.equal_ignore_case(last_seq_date, curr_seq_date): self.execute(''' update ro_global_config set gcfgvalue = current_date::text where gcfgcode = 'sys_seq_date_autoinc' ''') next_value = self.one_row( "select setval('sys_seq_date_autoinc', 1)").value_by_index( 0, 0, 1) return '{0}-{1}'.format(curr_seq_date, CUtils.int_2_format_str(next_value, 3)) next_value = self.one_row( "select nextval('sys_seq_date_autoinc')").value_by_index( 0, 0, 1) return '{0}-{1}'.format(curr_seq_date, CUtils.int_2_format_str(next_value, 3)) else: return CUtils.any_2_str( self.one_row( "select nextval('sys_seq_autoinc')").value_by_index( 0, 0, 1))
def give_me_creator(cls, extract_type, object_id: str, object_name: str, file_info: CDMFilePathInfoEx, file_content: CVirtualContent): input_parser_type = CUtils.any_2_str(extract_type) if CUtils.equal_ignore_case(input_parser_type, cls.BrowseEngine_Raster): return CViewCreatorRaster(object_id, object_name, file_info, file_content) elif CUtils.equal_ignore_case(input_parser_type, cls.BrowseEngine_Vector): return CViewCreatorVector(object_id, object_name, file_info, file_content) elif CUtils.equal_ignore_case(input_parser_type, cls.BrowseEngine_Document): return CViewCreatorDocument(object_id, object_name, file_info, file_content) else: return CParserCustom(object_id, object_name, file_info)