def __prepare_where_condition(self, column: CColumn, sql_text: str,
sql_params: dict):
column_type = self.__database.db_column_type_by_name(
column.db_column_type)
if CUtils.equal_ignore_case(column_type.set_value_method,
self.DB_Column_Set_Method_Param):
sql_text = CUtils.str_append(
sql_text, '{0}=:{1}'.format(column.name,
self.__param_name(column.name)),
' and ')
sql_params[self.__param_name(column.name)] = CUtils.any_2_str(
CUtils.dict_value_by_name(column.value, self.Name_Text, ''))
elif CUtils.equal_ignore_case(column_type.set_value_method,
self.DB_Column_Set_Method_Function):
sql_text = CUtils.str_append(
sql_text, '{0}=:{1}'.format(column.name,
self.__param_name(column.name)),
' and ')
sql_params[self.__param_name(
column.name)] = CUtils.replace_placeholder(
column_type.set_value_template,
dict({
self.Name_Value,
CUtils.any_2_str(
CUtils.dict_value_by_name(column.value,
self.Name_Text, ''))
}))
return sql_text, sql_params
def txt_to_xml(self, file_metadata_name_with_path: str):
"""
完成 王学谦 txt文件转xml,在函数外提前定义xml对象并获取父节点传入，函数会将通过父节点构造xml对象 by王学谦
:param file_metadata_name_with_path:查询的mat文件全名，带路径
:return xml_obj:将文件内容存储好的项目对象
"""
text_list = CFile.file_2_list(
file_metadata_name_with_path) # 获取mat文件作为列表
if (text_list is None) or len(text_list) == 0:
raise Exception('元数据文件无法读取，请检查') # 如果获取的文件内容为空，则抛出异常
xml_obj = CXml() # 建立xml对象
node_root = xml_obj.new_xml('root')
xml_obj.set_attr(node_root, self.Name_Type,
self.Transformer_TXT) # 设置root节点与属性
for row_text in text_list:
if not CUtils.equal_ignore_case(row_text, ''):
row_list = re.split(r'=',
row_text.strip()) # 利用正则表达式，根据一个或多个tab剪切字符
# item节点
node_item = xml_obj.create_element(node_root, 'item')
xml_obj.set_attr(node_item, self.Name_Name,
CUtils.any_2_str(row_list[0]).strip())
if len(row_list) > 1:
item_value = CUtils.any_2_str(row_list[1].strip())
if item_value.startswith('"') and item_value.endswith('"'):
item_value = item_value[1:-1]
xml_obj.set_element_text(node_item,
item_value) # 设置item节点与属性与内容
return xml_obj
def xls_to_xml(self, file_metadata_name_with_path: str):
"""
完成 王学谦 xls/xlsx文件转xml,在函数外提前定义xml对象并获取父节点传入，函数会将通过父节点构造xml对象 by王学谦
:param file_metadata_name_with_path:查询的xls/xlsx文件全名，带路径
:return xml_obj:将文件内容存储好的项目对象
"""
all_data = xlrd.open_workbook(file_metadata_name_with_path) # 获取表格所有内容
table_data = all_data.sheets()[0] # 默认获取第一个表格
cols_num = table_data.ncols # 获取列数
rows_num = table_data.nrows # 获取行数
cols_index = 0 # 预定义列的index
if CUtils.equal_ignore_case(CUtils.any_2_str(cols_num),
CUtils.any_2_str(2)):
pass # 无序号列从1列开始
elif CUtils.equal_ignore_case(CUtils.any_2_str(cols_num),
CUtils.any_2_str(3)):
cols_index = 1 # 有序号列从2列开始
else:
raise Exception('xls格式异常，无法正常解析')
xml_obj = CXml() # 建立xml对象
node_root = xml_obj.new_xml('root')
xml_obj.set_attr(node_root, self.Name_Type,
self.transformer_type) # 设置root节点与属性
for row in range(0, rows_num):
node_item = xml_obj.create_element(node_root, 'item')
xml_obj.set_attr(
node_item, self.Name_Name,
CUtils.any_2_str(table_data.cell(row,
cols_index).value).lower())
xml_obj.set_element_text(
node_item,
table_data.cell(row, cols_index + 1).value) # 设置item节点与属性与内容
return xml_obj
def mat_to_xml(self, file_metadata_name_with_path: str):
"""
完成 王学谦 mat文件转xml,在函数外提前定义xml对象并获取父节点传入，函数会将通过父节点构造xml对象 by王学谦
:param file_metadata_name_with_path:查询的mat文件全名，带路径
:return xml_obj:将文件内容存储好的项目对象
"""
text_list = CFile.file_2_list(
file_metadata_name_with_path) # 获取mat文件作为列表
if (text_list is
None) or len(text_list) == 0 or CUtils.equal_ignore_case(
CUtils.any_2_str(text_list), ''):
raise # 如果获取的文件内容为空，则抛出异常
flag = False # 设置标志
xml_obj = CXml() # 建立xml对象
node_root = xml_obj.new_xml('root')
xml_obj.set_attr(node_root, self.Name_Type,
self.transformer_type) # 设置root节点与属性
for index, row_text in enumerate(text_list):
if row_text.startswith('1\t'): # 从开头为1+tab键的行开始录入
flag = True
row_list = re.split(r'\s+', row_text) # 利用正则表达式，根据一个或多个tab剪切字符
if flag:
node_item = xml_obj.create_element(node_root, 'item')
xml_obj.set_attr(node_item, self.Name_Name,
CUtils.any_2_str(row_list[1]).lower())
xml_obj.set_element_text(
node_item,
CUtils.any_2_str(row_list[2].strip())) # 设置item节点与属性与内容
if not flag:
raise Exception('文件内容异常，无法正常识别文件开头') # 如果未找到1+tab键开头，则抛出异常
return xml_obj
def join_file(cls, path: str, *paths: AnyStr) -> str:
result = CUtils.any_2_str(path)
for each_path in paths:
real_file_name = CUtils.any_2_str(each_path)
if real_file_name.startswith(r'/') or real_file_name.startswith('\\'):
real_file_name = real_file_name[1:len(real_file_name)]
result = '{0}{1}{2}'.format(result, cls.sep(), real_file_name)
return result
def db_check_and_update(self, ib_id):
"""
检查并更新dm2_storage_directory表中记录
:return:
"""
if not self.ds_file_or_path.is_empty():
# 如果记录已经存在
db_path_modify_time = self.ds_file_or_path.value_by_name(
0, 'dsddirlastmodifytime', '')
if CUtils.equal_ignore_case(
CUtils.any_2_str(db_path_modify_time),
CUtils.any_2_str(self.file_modify_time)):
CLogger().info(
'目录[{0}]的最后修改时间, 和库中登记的没有变化, 子目录将被设置为忽略刷新! '.format(
self.file_name_with_full_path))
CFactory().give_me_db(self.db_server_id).execute(
'''
update dm2_storage_directory
set dsdScanStatus = 0, dsdScanFileStatus = 0, dsd_directory_valid = -1, dsd_ib_id = :ib_id, dsdscanmemo = :message
where dsdid = :dsdid
''', {
'dsdid':
self.my_id,
'ib_id':
ib_id,
'message':
'目录[{0}]的最后修改时间, 和库中登记的没有变化, 子目录将被设置为忽略刷新! '.format(
self.file_name_with_full_path)
})
else:
CLogger().info(
'目录[{0}]的最后修改时间, 和库中登记的有变化, 子目录将被设置为重新刷新! '.format(
self.file_name_with_full_path))
CFactory().give_me_db(self.db_server_id).execute(
'''
update dm2_storage_directory
set dsdScanStatus = 1, dsdScanFileStatus = 1, dsd_directory_valid = -1, dsd_ib_id = :ib_id, dsdscanmemo = :message
where dsdid = :dsdid
''', {
'dsdid':
self.my_id,
'ib_id':
ib_id,
'message':
'目录[{0}]的最后修改时间, 和库中登记的有变化, 子目录将被设置为重新刷新! '.format(
self.file_name_with_full_path)
})
else:
CLogger().info('目录[{0}]未在库中登记, 系统将登记该记录! '.format(
self.file_name_with_full_path))
self.__db_insert(ib_id)
def get_mission_seize_sql(self) -> str:
if self.__application_id is None:
return '''
update sch_center_mission
set scmprocessid = '{0}', scmstatus = 2
where scmid = (
select scmid
from sch_center_mission
where scmstatus = 1 and (
scmcenterid in (select scid from sch_center where scserver is null)
or scmcenterid is null
)
limit 1
for update skip locked
)
'''.format(self.SYSTEM_NAME_MISSION_ID)
else:
return '''
update sch_center_mission
set scmprocessid = '{0}', scmstatus = 2
where scmid = (
select scmid
from sch_center_mission
where scmstatus = 1 and (
scmcenterid in (select scid from sch_center where scserver = '{1}')
)
limit 1
for update skip locked
)
'''.format(self.SYSTEM_NAME_MISSION_ID,
CUtils.any_2_str(self.__application_id))
def give_me_extractor(cls, extract_type, object_id: str, object_name: str,
file_info: CDMFilePathInfoEx,
file_content: CVirtualContent):
input_parser_type = CUtils.any_2_str(extract_type)
if CUtils.equal_ignore_case(input_parser_type,
cls.MetaDataEngine_Raster):
return CMDExtractorRaster(object_id, object_name, file_info,
file_content)
elif CUtils.equal_ignore_case(input_parser_type,
cls.MetaDataEngine_Vector):
return CMDExtractorVector(object_id, object_name, file_info,
file_content)
elif CUtils.equal_ignore_case(input_parser_type,
cls.MetaDataEngine_Document_Tika):
return CMDExtractorDocument_Tika(object_id, object_name, file_info,
file_content)
elif CUtils.equal_ignore_case(input_parser_type,
cls.MetaDataEngine_Document):
return CMDExtractorDocument(object_id, object_name, file_info,
file_content)
elif CUtils.equal_ignore_case(input_parser_type,
cls.MetaDataEngine_Spatial_Layer):
return CMDExtractorSpatialLayer(object_id, object_name, file_info,
file_content)
elif CUtils.equal_ignore_case(input_parser_type,
cls.MetaDataEngine_Attached_File):
return CMDExtractorAttachedFile(object_id, object_name, file_info,
file_content)
else:
return CParserCustom(object_id, object_name, file_info)
def object_uni_full_name_by_id(self, object_id: str):
result = CFactory().give_me_db(self.db_server_id).one_value(
'''
select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name
from dm2_storage_object, dm2_storage_file, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id
and dm2_storage_file.dsfstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{0}'
and dm2_storage_object.dsoid = :object_id
union
select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name
from dm2_storage_object, dm2_storage_directory, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
and dm2_storage_directory.dsdstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{1}'
and dm2_storage_object.dsoid = :object_id
'''.format(self.FileType_File, self.FileType_Dir)
, {'object_id': object_id}
)
if result is not None:
return result
ds_object_info = CFactory().give_me_db(self.db_server_id).one_row(
'''
select dsoobjectname, dsoparentobjid
from dm2_storage_object
where dsoid = :object_id and dsodatatype not in ('file', 'dir')
'''
)
if ds_object_info.is_empty():
return None
object_name = ds_object_info.value_by_index(0, 0, '')
object_parent_id = ds_object_info.value_by_index(0, 1, '')
if CUtils.equal_ignore_case(object_parent_id, ''):
return None
result = CFactory().give_me_db(self.db_server_id).one_value(
'''
select dm2_storage.dstunipath || dm2_storage_file.dsffilerelationname as full_name
from dm2_storage_object, dm2_storage_file, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_file.dsf_object_id
and dm2_storage_file.dsfstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{0}'
and dm2_storage_object.dsoid = :object_id
union
select dm2_storage.dstunipath || dm2_storage_directory.dsddirectory as full_name
from dm2_storage_object, dm2_storage_directory, dm2_storage
where dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
and dm2_storage_directory.dsdstorageid = dm2_storage.dstid
and dm2_storage_object.dsodatatype = '{1}'
and dm2_storage_object.dsoid = :object_id
'''.format(self.FileType_File, self.FileType_Dir)
, {'object_id': object_parent_id}
)
if result is not None:
return CFile.join_file(CUtils.any_2_str(result), object_name)
else:
return None
def should_stop(self) -> bool:
if self.__application_id is None:
sql = '''
select scmid
from sch_center_mission
where (scmcommand <> '{0}' or scmstatus <> {1})
and (
(scmcenterid is null) or
(scmcenterid in (select scid from sch_center where scserver is null))
)
limit 1
'''.format(self.CMD_SHUTDOWN, self.Status_Finish)
else:
sql = '''
select scmid
from sch_center_mission
where (scmcommand <> '{0}' or scmstatus <> {1})
and (scmcenterid in (select scid from sch_center where scserver = '{2}'))
limit 1
'''.format(self.CMD_SHUTDOWN, self.Status_Finish,
CUtils.any_2_str(self.__application_id))
try:
factory = CFactory()
db = factory.give_me_db()
return not db.if_exists(sql)
except:
return True
def process_mission(self, dataset) -> str:
ds_id = dataset.value_by_name(0, 'query_dir_id', '')
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
inbound_id = dataset.value_by_name(0, 'query_dir_ib_id', None)
ds_subpath = dataset.value_by_name(0, 'query_subpath', '')
ds_root_path = dataset.value_by_name(0, 'query_rootpath', '')
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(
dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo, ds_retry_times))
self.update_dir_status(ds_id, process_result,
self.ProcStatus_Error)
return process_result
if ds_subpath == '':
ds_subpath = ds_root_path
else:
ds_subpath = CFile.join_file(ds_root_path, ds_subpath)
# 将所有子目录, 文件的可用性, 都改为未知!!!!
self.init_file_or_subpath_valid_unknown(ds_id)
try:
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID
and position((dsddirectory || '{0}') in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule, {
'dsdStorageID': ds_storage_id,
'dsdDirectory': ds_subpath
})
ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '')
CLogger().debug('处理的目录为: {0}'.format(ds_subpath))
self.parser_file_or_subpath_of_path(dataset, ds_id, ds_subpath,
ds_rule_content, inbound_id)
result = CResult.merge_result(
self.Success, '目录为[{0}]下的文件和子目录扫描处理成功!'.format(ds_subpath))
self.update_dir_status(ds_id, result)
return result
except Exception as err:
result = CResult.merge_result(
self.Failure, '目录为[{0}]下的文件和子目录扫描处理出现错误!错误原因为: {1}'.format(
ds_subpath, err.__str__()))
self.update_dir_status(ds_id, result)
return result
finally:
self.exchange_file_or_subpath_valid_unknown2invalid(ds_id)
def __import_each_record(self, data_source: CDataSetSeqReader, data_target: CTable) -> str:
data_target.column_list.reset()
data_source_record = data_source.record_as_dict()
for column_index in range(data_target.column_list.size()):
column_obj = data_target.column_list.column_by_index(column_index)
column_name = column_obj.name
column_data_set_method = self.__find_column_data_set_method(column_obj, data_source)
if column_data_set_method is None:
continue
column_value_set_type = CUtils.dict_value_by_name(column_data_set_method, self.Name_Type, self.Name_Common)
if CUtils.equal_ignore_case(column_value_set_type, self.Name_SQL):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
column_obj.set_sql(column_value_template)
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Geometry):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
column_obj.set_geometry(
column_value_template,
CUtils.dict_value_by_name(column_data_set_method, self.Name_Srid, None)
)
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Array):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, None)
if isinstance(column_value_template, list):
column_obj.set_array(column_value_template)
else:
column_value_template = CUtils.any_2_str(column_value_template)
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template,
data_source_record)
column_obj.set_array_str(column_value_template)
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Null):
column_obj.set_null()
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_File):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
file_format = CUtils.dict_value_by_name(column_data_set_method, self.Name_Format, self.Name_Binary)
file_format = self.replace_placeholder(file_format)
file_format = CUtils.replace_placeholder(file_format, data_source.record_as_dict())
file_encoding = CUtils.dict_value_by_name(column_data_set_method, self.Name_Encoding,
self.Encoding_UTF8)
file_encoding = self.replace_placeholder(file_encoding)
file_encoding = CUtils.replace_placeholder(file_encoding, data_source_record)
column_obj.set_value_from_file(column_value_template, file_format, file_encoding)
else:
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
column_obj.set_value(column_value_template)
data_target.save_data()
return CResult.merge_result(self.Success, '数据入库成功!')
def package(output_relation_dir):
"""
编译根目录下的包括子目录里的所有py文件成pyc文件到新的文件夹下
:param output_relation_dir: 需编译的目录
:return:
"""
output_relation_dir = CFile.unify(CUtils.any_2_str(output_relation_dir))
application_dir = CFile.file_path(CFile.file_abs_path(__file__))
output_dir = CFile.file_abs_path(CFile.join_file(application_dir, output_relation_dir))
for each_directory, dir_name_list, file_name_without_path_list in os.walk(application_dir):
directory_source = each_directory
directory_name = CFile.file_name(directory_source)
directory_relation = CFile.file_relation_path(each_directory, application_dir)
directory_target = CFile.join_file(output_dir, directory_relation)
path_deploy_enable = deploy_match_pattern_list(directory_relation, 'path.white_list', True, True)
path_deploy_enable = path_deploy_enable and deploy_match_pattern_list(directory_relation, 'path.black_list',
False, True)
if path_deploy_enable:
directory_deploy_enable = deploy_match_pattern_list(directory_name, 'directory.white_list', True, True)
directory_deploy_enable = directory_deploy_enable and deploy_match_pattern_list(
directory_name, 'directory.black_list',
False, True
)
if directory_deploy_enable:
for file_name_without_path in file_name_without_path_list:
file_deploy_enable = deploy_match_pattern_list(
file_name_without_path, 'file.white_list', True,
True
)
file_deploy_enable = file_deploy_enable and deploy_match_pattern_list(
file_name_without_path, 'file.black_list',
False, True
)
file_name_with_path_source = CFile.join_file(directory_source, file_name_without_path)
if file_deploy_enable:
file_compile_enable = deploy_match_pattern_list(
file_name_without_path, 'compile.file.white_list',
True, False
)
if file_compile_enable:
file_compile_enable = deploy_match_pattern_list(
file_name_without_path, 'compile.file.black_list',
False, False
)
file_name_without_path_target = CFile.change_file_ext(file_name_without_path, 'pyc')
file_name_with_path_target = CFile.join_file(directory_target, file_name_without_path_target)
CFile.check_and_create_directory_itself(directory_target)
if file_compile_enable:
py_compile.compile(file_name_with_path_source, cfile=file_name_with_path_target)
print('{0}-compile-success'.format(file_name_with_path_source))
else:
CFile.copy_file_to(file_name_with_path_source, directory_target)
print('{0}-no_compile'.format(file_name_with_path_source))
def record_as_dict(self) -> dict:
data_record = self._data.record(self.__record_index)
result = dict()
key_list = data_record.keys()
for key in key_list:
result[CUtils.any_2_str(key).lower()] = CUtils.dict_value_by_name(
data_record, key, None, False)
return result
def json_set_attr(cls, json_text, attr_name: str, attr_value) -> str:
new_result = CJson()
rt_json_text = CUtils.any_2_str(json_text)
if not CUtils.equal_ignore_case(rt_json_text, ''):
new_result.load_json_text(rt_json_text)
new_result.set_value_of_name(attr_name, attr_value)
return new_result.to_json()
def set_element_text(cls, element, text):
"""
设置一个节点的文本
:param element:
:param text:
:return:
"""
if element is not None:
element.text = etree.CDATA(CUtils.any_2_str(text))
def list_item_match(self, data_sample_list, keyword_value, fuzzy_matching):
"""
根据给定列表和关键字和匹配选项, 对列表是否包含关键字进行完全匹配或模糊匹配
:param data_sample_list:
:param keyword_value:
:param fuzzy_matching:
:return:
"""
keyword_text = CUtils.any_2_str(keyword_value).lower()
if fuzzy_matching:
for data_sample in data_sample_list:
data_sample_text = CUtils.any_2_str(data_sample).lower()
if data_sample_text.find(keyword_text) > -1:
return True
else:
if CUtils.list_count(data_sample_list, keyword_value) > 0:
return True
return False
def set_metadata_time(self,
result: int,
memo: str,
time_attr_name=None,
time_attr_value=None):
self.__metadata_time_extract_result__ = result
self.__metadata_time_extract_memo__ = memo
if time_attr_name is not None:
self.__time_information__.set_value_of_name(
CUtils.any_2_str(time_attr_name), time_attr_value)
def txt_to_xml(self, file_metadata_name_with_path: str):
"""
完成 王学谦 txt文件转xml,在函数外提前定义xml对象并获取父节点传入，函数会将通过父节点构造xml对象 by王学谦
:param file_metadata_name_with_path:查询的mat文件全名，带路径
:return xml_obj:将文件内容存储好的项目对象
"""
text_list = CFile.file_2_list(file_metadata_name_with_path) # 获取mat文件作为列表
if (text_list is None) or len(text_list) == 0:
raise Exception('元数据文件无法读取，请检查') # 如果获取的文件内容为空，则抛出异常
xml_obj = CXml() # 建立xml对象
node_root = xml_obj.new_xml('root')
xml_obj.set_attr(node_root, self.Name_Type, self.Transformer_TXT) # 设置root节点与属性
# 设置操作的节点
current_node = node_root
for row_text in text_list:
if CUtils.equal_ignore_case('row_text', 'END'):
break
# 分割字符
row_list = re.split(r'=', row_text.strip())
if len(row_list) >= 2:
# 为GROUP建立新节点
if CUtils.equal_ignore_case(row_list[0], 'GROUP'):
node_item = xml_obj.create_element(current_node, 'item')
# 值设为属性
item_value = CUtils.any_2_str(row_list[1].strip())
if item_value.startswith('"') and item_value.endswith('"'):
item_value = item_value[1:-1]
xml_obj.set_attr(node_item, self.Name_Name, item_value)
current_node = node_item
elif CUtils.equal_ignore_case(row_list[0], 'END_GROUP'):
current_node = xml_obj.node_xpath_one(current_node, '..')
else:
node_item = xml_obj.create_element(current_node, 'item')
xml_obj.set_attr(node_item, self.Name_Name, row_list[0].strip())
item_value = CUtils.any_2_str(row_list[1].strip())
if item_value.startswith('"') and item_value.endswith('"'):
item_value = item_value[1:-1]
xml_obj.set_element_text(node_item, item_value)
return xml_obj
def load_xml(self, xml_content: str):
"""
通过给定的xml内容, 对xml对象进行初始化
:param xml_content:
:return:
"""
if not CUtils.equal_ignore_case(CUtils.any_2_str(xml_content), ''):
parser = etree.XMLParser(remove_blank_text=True, strip_cdata=False)
self.__xml_root_node = etree.XML(xml_content.strip(), parser)
self.__xml_tree = etree.ElementTree(self.__xml_root_node)
def set_attr(cls, element, attr_name, attr_value):
"""
设置一个节点的属性
:param element:
:param attr_name:
:param attr_value:
:return:
"""
if element is not None:
element.set(attr_name, CUtils.any_2_str(attr_value))
def get_wkt_to_transform_coordinates(self, metadata_bus_dict,
metadata_bus_xml,
multiple_metadata_bus_filename_dict):
list_result = metadata_bus_xml.xpath(
'/product/imageAttributes/geographicInformation/geolocationGrid/imageTiePoint'
)
list_num = len(list_result)
temp_str = ''
temp_frist_flag = True
temp_frist_str = ''
for num in range(1, list_num + 1):
try:
latitude_node = metadata_bus_xml.xpath_one(
'/product/imageAttributes/geographicInformation/geolocationGrid'
'/imageTiePoint[{0}]/geodeticCoordinate/latitude'.format(
num))
latitude = metadata_bus_xml.get_element_text(latitude_node)
longitude_node = metadata_bus_xml.xpath_one(
'/product/imageAttributes/geographicInformation/geolocationGrid'
'/imageTiePoint[{0}]/geodeticCoordinate/longitude'.format(
num))
longitude = metadata_bus_xml.get_element_text(longitude_node)
if not CUtils.equal_ignore_case(
latitude, '') and not CUtils.equal_ignore_case(
longitude, ''):
temp_str = temp_str + '{0} {1},'.format(
CUtils.any_2_str(CUtils.to_decimal(latitude,
latitude)),
CUtils.any_2_str(
CUtils.to_decimal(longitude, longitude)))
if temp_frist_flag:
temp_frist_str = temp_str[:-1]
temp_frist_flag = False
except Exception as error:
raise Exception('计算wkt时出错,原因为{0}'.format(error.__str__()))
if CUtils.equal_ignore_case(temp_str, ''):
return ''
else:
temp_str = '{0}{1}'.format(temp_str, temp_frist_str)
wkt = 'POLYGON( ({0}) )'.format(temp_str)
return wkt
def __init__(self, job_id: str, job_params: str):
self.__id = job_id
self.__params = job_params
self.__mission_db_id = CUtils.any_2_str(
self.params_value_by_name(self.Job_Params_DB_Server_ID,
self.DB_Server_ID_Default))
self.__abnormal_job_retry_times = self.params_value_by_name(
self.Job_Params_Abnormal_Job_Retry_Times, -1)
if CUtils.equal_ignore_case(self.__abnormal_job_retry_times, -1):
self.__abnormal_job_retry_times = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_Parser_MetaData_Retry_Times,
self.Default_Abnormal_Job_Retry_Times)
self.custom_init()
def give_me_parser(cls, parser_type, object_id: str, object_name: str,
file_info: CDMFilePathInfoEx, file_custom_list: list):
input_parser_type = CUtils.any_2_str(parser_type)
if CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_Same_File_Main_Name):
return CDetailParser_Same_File_Main_Name(object_id, object_name,
file_info,
file_custom_list)
elif CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_File_Itself):
return CDetailParser_File_Itself(object_id, object_name, file_info,
file_custom_list)
elif CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_Directory_Itself):
return CDetailParser_Directory_Itself(object_id, object_name,
file_info, file_custom_list)
elif CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_File_Of_Same_Dir):
return CDetailParser_File_Of_Same_Dir(object_id, object_name,
file_info, file_custom_list)
elif CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_All_File_Of_Same_Dir):
return CDetailParser_All_File_Of_Same_Dir(object_id, object_name,
file_info,
file_custom_list)
elif CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_File_Of_Dir):
return CDetailParser_File_Of_Dir(object_id, object_name, file_info,
file_custom_list)
elif CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_All_File_Of_Dir):
return CDetailParser_All_File_Of_Dir(object_id, object_name,
file_info, file_custom_list)
elif CUtils.equal_ignore_case(input_parser_type,
cls.DetailEngine_Fuzzy_File_Main_Name):
return CDetailParser_Fuzzy_File_Main_Name(object_id, object_name,
file_info,
file_custom_list)
elif CUtils.equal_ignore_case(
input_parser_type,
cls.DetailEngine_Busdataset): # 用于入数据集的附属文件metadata.21at
return CDetailParser_Busdataset(object_id, object_name, file_info,
file_custom_list)
else:
# 注意, 这里改为基类了, 因为基类中将默认的处理清除已有附属文件的逻辑
return CDetailParser(object_id, object_name, file_info,
file_custom_list)
def process_metadata_table(self):
object_table_id = self._obj_id # 获取oid
object_table_data = self._dataset
metadata_bus_dict = self.get_metadata_bus_dict()
metadata_table_name = CUtils.dict_value_by_name(
self.information(), 'metadata_table_name', 'ap_product_metadata')
metadata_table = CTable()
metadata_table.load_info(self._db_id, metadata_table_name)
metadata_table.column_list.column_by_name('id').set_value(
object_table_id)
metadata_table.column_list.column_by_name('fid').set_value(
object_table_id)
dsometadataxml = object_table_data.value_by_name(
0, 'dsometadataxml_bus', None)
metadata_table.column_list.column_by_name('metaxml').set_value(
dsometadataxml)
# otherxml = CUtils.dict_value_by_name(metadata_bus_dict, 'otherxml', None)
# if not CUtils.equal_ignore_case(otherxml, ''):
# view_path = settings.application.xpath_one(self.Path_Setting_MetaData_Dir_View, None)
# browser_path = CFile.file_path(self._dataset.value_by_name(0, 'dso_browser', None))
# file_list = CFile.file_or_dir_fullname_of_path(
# CFile.join_file(view_path, browser_path), False, otherxml, CFile.MatchType_Regex
# )
# if len(file_list) > 0:
# metadata_table.column_list.column_by_name('otherxml').set_value(
# CFile.file_2_str(file_list[0])
# )
metadata_table.column_list.column_by_name('otherxml').set_value(
'<?xml version="1.0" encoding="UTF-8"?><root><plugin name="{0}"/></root>'
.format(
CUtils.dict_value_by_name(metadata_bus_dict, 'satelliteid',
None)))
if not metadata_table.if_exists():
metadata_table.column_list.column_by_name('version').set_value(
'1.0')
now_time = CUtils.any_2_str(
datetime.datetime.now().strftime('%F %T'))
metadata_table.column_list.column_by_name('addtime').set_value(
now_time)
result = metadata_table.save_data()
return result
def mdb_to_xml(self, file_metadata_name_with_path: str):
"""
TODO 王学谦 mdb文件转xml,在函数外提前定义xml对象并获取父节点传入，函数会将通过父节点构造xml对象
:param file_metadata_name_with_path:查询的mdb文件全名，带路径
:return xml_obj:将文件内容存储好的项目对象
"""
conn = None # 预定义连接与游标，方便释放
cur = None
try:
conn = self.get_mdb_connect(file_metadata_name_with_path)
cur = conn.cursor() # 游标
xml_obj = CXml() # 建立xml对象
node_root = xml_obj.new_xml('root')
xml_obj.set_attr(node_root, self.Name_Type, self.transformer_type) # 设置root节点与属性
table_name_list = ['mbii', 'mpid', 'mppi', 'mqc1', 'mqc2', 'mdac']
for table_name in table_name_list:
try:
sql = "SELECT * FROM " + table_name
cur.execute(sql)
table_data = cur.fetchall()
# total_rows = len(alldata) # 行
# total_cols = len(alldata[0]) # 列
node_property = xml_obj.create_element(node_root, 'property')
xml_obj.set_attr(node_property, 'tablename', table_name) # 设置property节点与属性与内容
for field_index, row_obj in enumerate(cur.description):
row_name = row_obj[0] # 字段名称
row_type = row_obj[1] # 字段类型
if row_type is bytearray: # 跳过长二进制数组
continue
node_item = xml_obj.create_element(node_property, 'item')
xml_obj.set_attr(node_item, self.Name_Name, CUtils.any_2_str(row_name).lower())
xml_obj.set_element_text(node_item, table_data[0][field_index]) # 设置item节点与属性与内容
except:
continue
except Exception as error:
raise Exception(error.__str__())
finally:
if cur is not None:
cur.close()
if conn is not None:
conn.close()
return xml_obj
def give_me_db(self, db_id=None) -> CDataBase:
if db_id is None:
rt_db_id = self.DB_Server_ID_Default
else:
rt_db_id = CUtils.any_2_str(db_id)
if CUtils.equal_ignore_case(rt_db_id, ''):
rt_db_id = self.DB_Server_ID_Default
databases = settings.application.xpath_one(self.Name_DataBases, None)
if databases is None:
raise Exception('系统未配置数据库定义, 请检查修正后重试！')
for database in databases:
if rt_db_id == CUtils.dict_value_by_name(database, self.Name_ID, self.DB_Server_ID_Default):
return self.create_db(database)
raise Exception('未找到标示为[{0}]]的数据库定义！'.format(rt_db_id))
def get_abnormal_mission_restart_sql(self) -> str:
if self.__application_id is None:
return '''
update sch_center_mission
set scmstatus = 1, scmprocessid = null
where scmstatus = 2
and (
(scmcenterid is null) or
(scmcenterid in (select scid from sch_center where scserver is null))
)
'''
else:
return '''
update sch_center_mission
set scmstatus = 1, scmprocessid = null
where scmstatus = 2
and (scmcenterid in (select scid from sch_center where scserver = '{0}'))
'''.format(CUtils.any_2_str(self.__application_id))
def seq_next_value(self, seq_type: int) -> str:
if seq_type == self.Seq_Type_Date_AutoInc:
sql_last_seq_date = '''
select coalesce(gcfgvalue, current_date::text) as last_date, current_date::text as curr_date from ro_global_config where
gcfgcode = 'sys_seq_date_autoinc'
'''
ds_last_seq_date = self.one_row(sql_last_seq_date)
if ds_last_seq_date.is_empty():
self.execute('''
insert into ro_global_config(gcfgid, gcfgcode, gcfgtitle, gcfgvalue, gcfgmemo)
values (10001, 'sys_seq_date_autoinc', '日期自增序列最后记录', current_date::text, null)
''')
ds_last_seq_date = self.one_row(sql_last_seq_date)
last_seq_date = ds_last_seq_date.value_by_name(
0, 'last_date', None)
curr_seq_date = ds_last_seq_date.value_by_name(
0, 'curr_date', None)
if last_seq_date is None:
self.execute('''
update ro_global_config set gcfgvalue = current_date::text
where gcfgcode = 'sys_seq_date_autoinc'
''')
elif not CUtils.equal_ignore_case(last_seq_date, curr_seq_date):
self.execute('''
update ro_global_config set gcfgvalue = current_date::text
where gcfgcode = 'sys_seq_date_autoinc'
''')
next_value = self.one_row(
"select setval('sys_seq_date_autoinc', 1)").value_by_index(
0, 0, 1)
return '{0}-{1}'.format(curr_seq_date,
CUtils.int_2_format_str(next_value, 3))
next_value = self.one_row(
"select nextval('sys_seq_date_autoinc')").value_by_index(
0, 0, 1)
return '{0}-{1}'.format(curr_seq_date,
CUtils.int_2_format_str(next_value, 3))
else:
return CUtils.any_2_str(
self.one_row(
"select nextval('sys_seq_autoinc')").value_by_index(
0, 0, 1))
def give_me_creator(cls, extract_type, object_id: str, object_name: str,
file_info: CDMFilePathInfoEx,
file_content: CVirtualContent):
input_parser_type = CUtils.any_2_str(extract_type)
if CUtils.equal_ignore_case(input_parser_type,
cls.BrowseEngine_Raster):
return CViewCreatorRaster(object_id, object_name, file_info,
file_content)
elif CUtils.equal_ignore_case(input_parser_type,
cls.BrowseEngine_Vector):
return CViewCreatorVector(object_id, object_name, file_info,
file_content)
elif CUtils.equal_ignore_case(input_parser_type,
cls.BrowseEngine_Document):
return CViewCreatorDocument(object_id, object_name, file_info,
file_content)
else:
return CParserCustom(object_id, object_name, file_info)
