def test_file_info_list(self):
return [
{
self.Name_Test_File_Type:
self.FileType_Dir,
self.Name_Test_file_path:
'gdb数据集{0}FileGeodb.gdb'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'FileGeodb'
}, {
self.Name_Test_File_Type:
self.FileType_Dir,
self.Name_Test_file_path:
'gdb数据集{0}FileGeodb2.gdb'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'FileGeodb2'
}
# ,
# {
# self.Name_Test_File_Type: self.FileType_Dir,
# self.Name_Test_file_path: 'gdb数据集{0}FileGeoNone.gdb'.format(CFile.sep()),
# self.Name_Test_object_confirm: self.Object_Confirm_IKnown,
# self.Name_Test_object_name: 'FileGeoNone'
# }
]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828F.img'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'BJ21_002E74VI_001_0120200730014001_00520200828F'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828M.img'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828P.img'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828T.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828Y.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type: self.FileType_Dir,
self.Name_Test_file_path: '202008{0}参考dom'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown,
self.Name_Test_object_name: '参考dom'
}]
def process_mission(self, dataset) -> str:
ds_id = dataset.value_by_name(0, 'query_dir_id', '')
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
inbound_id = dataset.value_by_name(0, 'query_dir_ib_id', None)
ds_subpath = dataset.value_by_name(0, 'query_subpath', '')
ds_root_path = dataset.value_by_name(0, 'query_rootpath', '')
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(
dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo, ds_retry_times))
self.update_dir_status(ds_id, process_result,
self.ProcStatus_Error)
return process_result
if ds_subpath == '':
ds_subpath = ds_root_path
else:
ds_subpath = CFile.join_file(ds_root_path, ds_subpath)
# 将所有子目录, 文件的可用性, 都改为未知!!!!
self.init_file_or_subpath_valid_unknown(ds_id)
try:
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID
and position((dsddirectory || '{0}') in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule, {
'dsdStorageID': ds_storage_id,
'dsdDirectory': ds_subpath
})
ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '')
CLogger().debug('处理的目录为: {0}'.format(ds_subpath))
self.parser_file_or_subpath_of_path(dataset, ds_id, ds_subpath,
ds_rule_content, inbound_id)
result = CResult.merge_result(
self.Success, '目录为[{0}]下的文件和子目录扫描处理成功!'.format(ds_subpath))
self.update_dir_status(ds_id, result)
return result
except Exception as err:
result = CResult.merge_result(
self.Failure, '目录为[{0}]下的文件和子目录扫描处理出现错误!错误原因为: {1}'.format(
ds_subpath, err.__str__()))
self.update_dir_status(ds_id, result)
return result
finally:
self.exchange_file_or_subpath_valid_unknown2invalid(ds_id)
def test_file_info_list(self):
return [
{
self.Name_Test_File_Type: self.FileType_File,
self.Name_Test_file_path: '北京市20200101自由分幅测试{0}影像{0}b.tif'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown,
self.Name_Test_object_name: 'b'
}
]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020A.TIF'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'J50E005015BJ208M2020A'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020A.TWF'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020P.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020M.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}]
def test_file_info_list(self):
return [
{
self.Name_Test_File_Type: self.FileType_File,
self.Name_Test_file_path: '202008{0}镶嵌影像成果{0}区域镶嵌{0}1990天津任意直角坐标系{0}tianjin_120116滨海新区_202008_Bj2_08m_tj1990.img'.format(
CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown,
self.Name_Test_object_name: 'tianjin_120116滨海新区_202008_Bj2_08m_tj1990'
}
]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}镶嵌影像成果{0}区域镶嵌{0}其他国家标准坐标系{0}a.img'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'a'
}]
def process(self) -> str:
"""
完成 负责人 张源博、赵宇飞 在这里提取影像数据的快视图, 将元数据文件存储在self.file_content.view_root_dir下
注意返回的串中有快视图和拇指图的文件名
注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!!
:return:
"""
# 获取对象类型
type = 'default'
group = 'default'
catalog = 'default'
# 构建数据对象object对应的识别插件，获取get_information里面的信息
class_classified_obj = CObject.get_plugins_instance_by_object_id(self.file_info.db_server_id, self.object_id)
if class_classified_obj is not None:
plugins_info = class_classified_obj.get_information()
type = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Type, 'default')
group = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Group, 'default')
catalog = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Catalog, 'default')
create_time = CTime.today()
create_format_time = CTime.format_str(create_time, '%Y%m%d')
year = CTime.format_str(create_time, '%Y')
month = CTime.format_str(create_time, '%m')
day = CTime.format_str(create_time, '%d')
sep = CFile.sep() # 操作系统的不同处理分隔符不同
sep_list = [catalog, group, type, year, month, day]
relative_path_part = sep.join(sep_list) # 相对路径格式
view_relative_path_browse = r'{2}{0}{2}{1}_browse.png'.format(relative_path_part, self.object_id, sep)
view_relative_path_thumb = r'{2}{0}{2}{1}_thumb.jpg'.format(relative_path_part, self.object_id, sep)
view_relative_path_geotiff = r'{2}{0}{2}{1}_browse.tiff'.format(relative_path_part, self.object_id, sep)
browse_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_browse)
thumb_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_thumb)
geotiff_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_geotiff)
# 进程调用模式
json_out_view = CJson()
json_out_view.set_value_of_name('image_path', self.file_info.file_name_with_full_path)
json_out_view.set_value_of_name('browse_full_path', browse_full_path)
json_out_view.set_value_of_name('thumb_full_path', thumb_full_path)
json_out_view.set_value_of_name('geotiff_full_path', geotiff_full_path)
result_view = CProcessUtils.processing_method(self.create_view_json, json_out_view)
# result_view = self.create_view(self.file_info.file_name_with_full_path, browse_full_path, thumb_full_path,
# geotiff_full_path)
# result_view = self.create_view_json(json_out_view)
if CResult.result_success(result_view):
result = CResult.merge_result(self.Success, '处理完毕!')
result = CResult.merge_result_info(result, self.Name_Browse, view_relative_path_browse)
result = CResult.merge_result_info(result, self.Name_Thumb, view_relative_path_thumb)
result = CResult.merge_result_info(result, self.Name_Browse_GeoTiff, view_relative_path_geotiff)
else:
result = result_view
return result
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_Dir,
self.Name_Test_file_path:
'202008{0}信息产品{0}2020年土地执法项目{0}矢量成果'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'矢量成果'
}]
def db_update_status_on_path_invalid(self):
"""
处理目录不存在时的业务
1. 标记已经入库的文件为无效
1. 标记已经入库的子目录为无效
:return:
"""
path_name_with_relation_path = CFile.join_file(
self.file_path_with_rel_path, '')
params = dict()
params['dsdStorageID'] = self.storage_id
params['dsdSubDirectory'] = path_name_with_relation_path
sql_update_file_invalid = '''
update dm2_storage_file
set dsffilevalid = 0, dsfscanstatus = 0
where dsfdirectoryid in (
select dsdid
from dm2_storage_directory
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1
)
'''.format(CFile.sep())
sql_update_path_invalid = '''
update dm2_storage_directory
set dsd_directory_valid = 0, dsdscanstatus = 0, dsdscanfilestatus = 0, dsdscandirstatus = 0
where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1
'''.format(CFile.sep())
engine = CFactory().give_me_db(self.db_server_id)
session = engine.give_me_session()
try:
engine.session_execute(session, sql_update_file_invalid, params)
engine.session_execute(session, sql_update_path_invalid, params)
engine.session_commit(session)
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
engine.session_rollback(session)
finally:
engine.session_close(session)
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}其他国家标准坐标系{0}2772.0-509.0.tif'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'2772.0-509.0'
}]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}镶嵌影像成果{0}市域镶嵌{0}2000国家标准坐标系{0}tianjin_202008_bj2_08m_cgcs2000.img'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'tianjin_202008_bj2_08m_cgcs2000'
}]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'影像{0}云南高分影像{0}F47{0}'
'F47E001007BJ210M2017A.MDB'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'F47E001007BJ210M2017A'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'影像{0}云南高分影像{0}F48{0}'
'F48E001001BJ210M2017A.MDB'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'F48E001001BJ210M2017A'
}]
def test_file_info_list(self):
return [
{
self.Name_Test_File_Type: self.FileType_Dir,
self.Name_Test_file_path: 'mbtiles{0}hdqimage201510_0.mbtiles'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown,
self.Name_Test_object_name: 'hdqimage201510_0'
},
{
self.Name_Test_File_Type: self.FileType_Dir,
self.Name_Test_file_path: 'mbtiles{0}hdqimage201510.xml'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name: None
}
]
def test_file_info_list(self):
return [
{
self.Name_Test_File_Type: self.FileType_File,
self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018M.xml'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name: None
}, {
self.Name_Test_File_Type: self.FileType_File,
self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018O.tfw'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name: None
}, {
self.Name_Test_File_Type: self.FileType_File,
self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018O.tif'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown,
self.Name_Test_object_name: 'G49G001038GF210M2018O'
}, {
self.Name_Test_File_Type: self.FileType_File,
self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018P.xml'.format(CFile.sep()),
self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name: None
}
]
def get_custom_affiliated_file_character(self):
file_path = self.file_info.file_path
file_main_name = self.file_info.file_main_name
regularexpression = '(?i)^' + file_main_name[:-1] + r'.\..*'
letter_location_FenFu = file_path.find('FenFu')
file_path_sub = file_path[letter_location_FenFu + 6:]
letter_location = file_path_sub.find(CFile.sep())
shp_path = CFile.join_file(file_path[:letter_location_FenFu + 5], file_path_sub[:letter_location])
return [
{
self.Name_FilePath: file_path, # 附属文件的路径
self.Name_RegularExpression: regularexpression, # 附属文件的匹配规则
# 应该从上面匹配到的文件剔除的文件的匹配规则
self.Name_No_Match_RegularExpression: '(?i)^' + file_main_name + r'\..*$'
}, {
self.Name_FilePath: shp_path, # 附属文件的路径
self.Name_RegularExpression: r'(?i)\.shp$', # 附属文件的匹配规则
# 应该从上面匹配到的文件剔除的文件的匹配规则
self.Name_No_Match_RegularExpression: None
}
]
def process_mission(self, dataset):
dso_id = dataset.value_by_name(0, 'dsoid', '')
dso_data_type = dataset.value_by_name(0, 'dsodatatype', '')
dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '')
dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '')
CLogger().debug('开始处理对象: {0}.{1}.{2}.{3}的元数据'.format(
dso_id, dso_data_type, dso_object_type, dso_object_name))
dso_object_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if dso_object_retry_times >= self.abnormal_job_retry_times():
dso_object_last_process_memo = CUtils.any_2_str(
dataset.value_by_name(0, 'dsometadataparsememo', None))
process_result = CResult.merge_result(
self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
dso_object_last_process_memo, dso_object_retry_times))
self.db_update_object_exception(dso_id, process_result,
self.ProcStatus_Error)
return process_result
ds_file_info = self.get_object_info(dso_id, dso_data_type)
if ds_file_info.value_by_name(0, 'query_object_valid',
self.DB_False) == self.DB_False:
process_result = CResult.merge_result(
self.Success, '文件或目录[{0}]不存在，元数据无法解析, 元数据处理正常结束!'.format(
ds_file_info.value_by_name(0, 'query_object_fullname',
'')))
self.db_update_object_status(dso_id, process_result)
return process_result
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule, {
'dsdStorageID':
ds_file_info.value_by_name(0, 'query_object_storage_id', ''),
'dsdDirectory':
ds_file_info.value_by_name(0, 'query_object_relation_path', '')
})
ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '')
file_info_obj = CDMFilePathInfoEx(
dso_data_type,
ds_file_info.value_by_name(0, 'query_object_fullname', ''),
ds_file_info.value_by_name(0, 'query_object_storage_id', ''),
ds_file_info.value_by_name(0, 'query_object_file_id', ''),
ds_file_info.value_by_name(0, 'query_object_file_parent_id', ''),
ds_file_info.value_by_name(0, 'query_object_owner_id', ''),
self.get_mission_db_id(), ds_rule_content)
plugins_obj = CPluginsMng.plugins(file_info_obj, dso_object_type)
if plugins_obj is None:
process_result = CResult.merge_result(
self.Failure, '文件或目录[{0}]的类型插件[{1}]不存在，元数据无法解析, 处理结束!'.format(
ds_file_info.value_by_name(0, 'query_object_fullname', ''),
dso_object_type))
self.db_update_object_status(dso_id, process_result,
self.ProcStatus_Error)
return process_result
plugins_obj.classified()
if not plugins_obj.create_virtual_content():
process_result = CResult.merge_result(
self.Failure, '文件或目录[{0}]的内容提取失败, 元数据无法提取!'.format(
ds_file_info.value_by_name(0, 'query_object_fullname',
'')))
self.db_update_object_status(dso_id, process_result)
return process_result
try:
metadata_parser = CMetaDataParser(dso_id, dso_object_name,
file_info_obj,
plugins_obj.file_content,
plugins_obj.get_information())
process_result = plugins_obj.parser_metadata(metadata_parser)
if CResult.result_success(process_result):
message = ''
step_success = (
metadata_parser.metadata.metadata_extract_result !=
self.DB_False)
if not step_success:
message = CUtils.str_append(message, '实体元数据解析出现错误', ', ')
all_step_success = step_success
step_success = metadata_parser.metadata.metadata_bus_extract_result != self.DB_False
if not step_success:
message = CUtils.str_append(message, '实体元数据解析出现错误', ', ')
all_step_success = all_step_success and step_success
step_success = metadata_parser.metadata.metadata_view_extract_result != self.DB_False
if not step_success:
message = CUtils.str_append(message, '快视图等可视化元数据解析出现错误',
', ')
all_step_success = all_step_success and step_success
step_success = metadata_parser.metadata.metadata_time_extract_result != self.DB_False
if not step_success:
message = CUtils.str_append(message, '时间元数据解析出现错误', ', ')
all_step_success = all_step_success and step_success
step_success = metadata_parser.metadata.metadata_spatial_extract_result != self.DB_False
if not step_success:
message = CUtils.str_append(message, '空间投影元数据解析出现错误', ', ')
all_step_success = all_step_success and step_success
if all_step_success:
self.db_update_object_status(dso_id, process_result)
return process_result
else:
process_result = CResult.merge_result(
self.Failure, message)
self.db_update_object_status(dso_id, process_result)
return process_result
else:
self.db_update_object_status(dso_id, process_result)
return process_result
except Exception as error:
process_result = CResult.merge_result(
self.Failure, '文件或目录[{0}]元数据解析过程出现异常! 错误原因为: {1}'.format(
ds_file_info.value_by_name(0, 'query_object_fullname', ''),
error.__str__()))
self.db_update_object_exception(dso_id, process_result)
return process_result
finally:
plugins_obj.destroy_virtual_content()
def process_mission(self, dataset):
dso_id = dataset.value_by_name(0, 'dsoid', '')
dso_data_type = dataset.value_by_name(0, 'dsodatatype', '')
dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '')
dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '')
CLogger().debug('开始处理对象: {0}.{1}.{2}.{3}的元数据'.format(
dso_id, dso_data_type, dso_object_type, dso_object_name))
ds_object_info = self.get_object_info(dso_id, dso_data_type)
ds_object_storage_option = ds_object_info.value_by_name(
0, 'query_object_storage_option', None)
if ds_object_info.value_by_name(0, 'query_object_valid',
self.DB_False) == self.DB_False:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsotagsparsestatus = 0
, dsolastmodifytime = now()
, dsotagsparsememo = '文件或目录不存在，标签无法解析'
where dsoid = :dsoid
''', {'dsoid': dso_id})
return CResult.merge_result(
self.Success, '文件或目录[{0}]不存在，标签处理正常结束!'.format(
ds_object_info.value_by_name(0, 'query_object_fullname',
'')))
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule, {
'dsdStorageID':
ds_object_info.value_by_name(0, 'query_object_storage_id', ''),
'dsdDirectory':
ds_object_info.value_by_name(0, 'query_object_relation_path',
'')
})
ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '')
file_info_obj = CDMFilePathInfoEx(
dso_data_type,
ds_object_info.value_by_name(0, 'query_object_fullname', ''),
ds_object_info.value_by_name(0, 'query_object_storage_id', ''),
ds_object_info.value_by_name(0, 'query_object_file_id', ''),
ds_object_info.value_by_name(0, 'query_object_file_parent_id', ''),
ds_object_info.value_by_name(0, 'query_object_owner_id', ''),
self.get_mission_db_id(), ds_rule_content)
plugins_obj = CPluginsMng.plugins(file_info_obj, dso_object_type)
if plugins_obj is None:
return CResult.merge_result(
self.Failure, '文件或目录[{0}]的类型插件[{1}]不存在，对象详情无法解析, 处理结束!'.format(
ds_object_info.value_by_name(0, 'query_object_fullname',
''), dso_object_type))
try:
plugins_information = plugins_obj.get_information()
tags_parser_rule = CUtils.dict_value_by_name(
plugins_information, plugins_obj.Plugins_Info_TagsEngine, None)
if tags_parser_rule is None:
tags_parser_rule = CJson.json_attr_value(
CUtils.any_2_str(ds_object_storage_option),
self.Path_Setting_MetaData_Tags_Rule, None)
if tags_parser_rule is None:
tags_parser_rule = settings.application.xpath_one(
self.Path_Setting_MetaData_Tags_Rule, None)
if tags_parser_rule is None:
process_result = CResult.merge_result(
CResult.Success, '系统未设置标签库和识别模式, 标签解析将自动结束')
else:
process_result = plugins_obj.parser_tags(
CTagsParser(dso_id, dso_object_name, file_info_obj,
dataset.value_by_name(0, 'dsoaliasname', ''),
tags_parser_rule))
self.db_update_object_status(dso_id, process_result)
return process_result
except Exception as err:
process_result = CResult.merge_result(
self.Failure, '文件或目录[{0}]对象业务分类解析过程出现错误! 错误原因为: {1}'.format(
ds_object_info.value_by_name(0, 'query_object_fullname',
''), err.__str__()))
self.db_update_object_status(dso_id, process_result)
return process_result
def process_main_table(self):
object_table_id = self._obj_id # 获取oid
object_table_data = self._dataset
metadata_bus_dict = self.get_metadata_bus_dict()
main_table_name = CUtils.dict_value_by_name(self.information(),
'main_table_name',
'ap_product')
main_table = CTable()
main_table.load_info(self._db_id, main_table_name)
main_table.column_list.column_by_name('id').set_value(object_table_id)
productname = CUtils.dict_value_by_name(metadata_bus_dict,
'productname', None)
if CUtils.equal_ignore_case(productname, ''):
productname = object_table_data.value_by_name(
0, 'dsoobjectname', None)
main_table.column_list.column_by_name('productname').set_value(
productname)
main_table.column_list.column_by_name('producttype').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'producttype', None))
main_table.column_list.column_by_name('regioncode').set_null()
main_table.column_list.column_by_name('productattribute').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'productattribute',
None))
centerlatitude = CUtils.dict_value_by_name(metadata_bus_dict,
'centerlatitude', None)
centerlongitude = CUtils.dict_value_by_name(metadata_bus_dict,
'centerlongitude', None)
centerlonlat = '{0},{1}'.format(centerlongitude, centerlatitude)
main_table.column_list.column_by_name('centerlonlat').set_value(
centerlonlat)
main_table.column_list.column_by_name('geomwkt').set_sql('''
st_astext(
(select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}')
)
'''.format(object_table_id))
main_table.column_list.column_by_name('geomobj').set_sql('''
(select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}')
'''.format(object_table_id))
main_table.column_list.column_by_name('browserimg').set_value(
'{0}{1}'.format(
CFile.sep(),
object_table_data.value_by_name(0, 'dso_browser', None)))
main_table.column_list.column_by_name('thumbimg').set_value(
'{0}{1}'.format(
CFile.sep(),
object_table_data.value_by_name(0, 'dso_thumb', None)))
main_table.column_list.column_by_name('publishdate').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'publishdate', None))
main_table.column_list.column_by_name('copyright').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'copyright', None))
dso_time = object_table_data.value_by_name(0, 'dso_time', None)
dso_time_json = CJson()
dso_time_json.load_obj(dso_time)
imgdate = dso_time_json.xpath_one('time', None)
if not CUtils.equal_ignore_case(imgdate, ''):
main_table.column_list.column_by_name('imgdate').set_value(
imgdate[0:4])
else:
main_table.column_list.column_by_name('imgdate').set_null()
main_table.column_list.column_by_name('starttime').set_value(
dso_time_json.xpath_one('start_time', None))
main_table.column_list.column_by_name('endtime').set_value(
dso_time_json.xpath_one('end_time', None))
resolution = CUtils.any_2_str(
CUtils.dict_value_by_name(metadata_bus_dict, 'resolution', None))
if not CUtils.equal_ignore_case(resolution, ''):
if '/' in resolution:
resolution_list = resolution.split('/')
temp_list = list()
for resolution in resolution_list:
temp_list.append(CUtils.to_decimal(resolution, -1))
temp_list = list(set(temp_list)) # 去重
if -1 in temp_list:
temp_list.remove(-1)
if len(temp_list) > 0:
main_table.column_list.column_by_name(
'resolution').set_value(min(temp_list))
else:
main_table.column_list.column_by_name(
'resolution').set_value(0)
else:
main_table.column_list.column_by_name('resolution').set_value(
resolution)
else:
main_table.column_list.column_by_name('resolution').set_value(0)
main_table.column_list.column_by_name('filesize').set_sql('''
(select sum(dodfilesize) from dm2_storage_obj_detail where dodobjectid='{0}')
'''.format(object_table_id))
productid = CUtils.dict_value_by_name(metadata_bus_dict, 'productid',
None)
if CUtils.equal_ignore_case(productid, ''):
object_type = object_table_data.value_by_name(0, 'dsodatatype', '')
if CUtils.equal_ignore_case(object_type, self.Name_Dir):
main_table.column_list.column_by_name('productid').set_value(
object_table_data.value_by_name(0, 'dsoobjectname', None))
elif CUtils.equal_ignore_case(object_type, self.Name_File):
main_table.column_list.column_by_name('productid').set_sql('''
(SELECT dsffilename FROM dm2_storage_file WHERE dsf_object_id = '{0}')
'''.format(object_table_id))
else:
main_table.column_list.column_by_name('productid').set_null()
else:
main_table.column_list.column_by_name('productid').set_value(
productid)
main_table.column_list.column_by_name('remark').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'remark', None))
main_table.column_list.column_by_name('extent').set_sql('''
(select dso_geo_bb_wgs84 from dm2_storage_object where dsoid='{0}')
'''.format(object_table_id))
main_table.column_list.column_by_name('proj').set_null() # 原始数据保持空
main_table.column_list.column_by_name('dataid').set_value(
object_table_id)
main_table.column_list.column_by_name('shplog').set_null()
if not main_table.if_exists():
now_time = CUtils.any_2_str(
datetime.datetime.now().strftime('%F %T'))
main_table.column_list.column_by_name('addtime').set_value(
now_time)
main_table.column_list.column_by_name('isdel').set_value(0)
main_table.column_list.column_by_name('projectnames').set_value(
'productname')
result = main_table.save_data()
return result
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020O.tif'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'272-500BJ208M2020O'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020O.TFW '
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020P.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020M.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020a.tif'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'272-501BJ208M2020a'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020B.tif'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020C.tif'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020D.tif'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020A.TFW '
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020P.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020M.XML'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.dbf'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.prj'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.sbn'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.sbx'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.shp'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.shp.xml'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IUnKnown,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.shx'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.dbf'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.prj'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.sbn'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.sbx'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.shp'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.shp.xml'
.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IUnKnown,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}正确数据{0}'
'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.shx'.
format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.dbf'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.prj'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.sbn'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.shp'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.sbx'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.shx'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.dbf'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.prj'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.sbn'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.sbx'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.shp'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.shx'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'矢量数据{0}错误数据{0}'
'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.shp.xml'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IUnKnown,
self.Name_Test_object_name:
None
}]
def process_ndi_table(self):
object_table_id = self._obj_id # 获取oid
metadata_bus_dict = self.get_metadata_bus_dict()
ndi_table_name = CUtils.dict_value_by_name(self.information(),
'ndi_table_name',
'ap_product_ndi')
ndi_table = CTable()
ndi_table.load_info(self._db_id, ndi_table_name)
ndi_table.column_list.column_by_name('id').set_value(object_table_id)
productname = CUtils.dict_value_by_name(metadata_bus_dict,
'productname', None)
if CUtils.equal_ignore_case(productname, ''):
productname = self._dataset.value_by_name(0, 'dsoobjectname', None)
ndi_table.column_list.column_by_name('rid').set_value(productname)
ndi_table.column_list.column_by_name('fid').set_value(object_table_id)
ndi_table.column_list.column_by_name('satelliteid').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'satelliteid', None))
ndi_table.column_list.column_by_name('sensorid').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'sensorid', None))
ndi_table.column_list.column_by_name('topleftlatitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'topleftlatitude',
None))
ndi_table.column_list.column_by_name('topleftlongitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'topleftlongitude',
None))
ndi_table.column_list.column_by_name('toprightlatitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'toprightlatitude',
None))
ndi_table.column_list.column_by_name('toprightlongitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'toprightlongitude',
None))
ndi_table.column_list.column_by_name('bottomrightlatitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'bottomrightlatitude',
None))
ndi_table.column_list.column_by_name('bottomrightlongitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict,
'bottomrightlongitude', None))
ndi_table.column_list.column_by_name('bottomleftlatitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'bottomleftlatitude',
None))
ndi_table.column_list.column_by_name('bottomleftlongitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'bottomleftlongitude',
None))
ndi_table.column_list.column_by_name('centerlatitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'centerlatitude',
None))
ndi_table.column_list.column_by_name('centerlongitude').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'centerlongitude',
None))
transformimg = CUtils.dict_value_by_name(metadata_bus_dict,
'transformimg', None)
if not CUtils.equal_ignore_case(transformimg, ''):
view_path = settings.application.xpath_one(
self.Path_Setting_MetaData_Dir_View, None)
browser_path = CFile.file_path(
self._dataset.value_by_name(0, 'dso_browser', None))
file_list = CFile.file_or_dir_fullname_of_path(
CFile.join_file(view_path, browser_path), False, transformimg,
CFile.MatchType_Regex)
if len(file_list) > 0:
ndi_table.column_list.column_by_name('transformimg').set_value(
'{0}{1}'.format(
CFile.sep(),
CFile.join_file(browser_path,
CFile.file_name(file_list[0]))))
ndi_table.column_list.column_by_name('filesize').set_sql('''
(select sum(dodfilesize) from dm2_storage_obj_detail where dodobjectid='{0}')
'''.format(object_table_id))
ndi_table.column_list.column_by_name('dataexist').set_value(0)
ndi_table.column_list.column_by_name('centertime').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'centertime', None))
resolution = CUtils.dict_value_by_name(metadata_bus_dict, 'resolution',
None)
if not CUtils.equal_ignore_case(resolution, ''):
ndi_table.column_list.column_by_name('resolution').set_value(
resolution)
else:
ndi_table.column_list.column_by_name('resolution').set_value(0)
rollangle = CUtils.dict_value_by_name(metadata_bus_dict, 'rollangle',
0)
if CUtils.equal_ignore_case(rollangle, ''):
rollangle = 0
ndi_table.column_list.column_by_name('rollangle').set_value(rollangle)
cloudpercent = CUtils.dict_value_by_name(metadata_bus_dict,
'cloudpercent', 0)
if CUtils.equal_ignore_case(cloudpercent, ''):
cloudpercent = 0
ndi_table.column_list.column_by_name('cloudpercent').set_value(
cloudpercent)
ndi_table.column_list.column_by_name('dataum').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'dataum', None))
ndi_table.column_list.column_by_name('acquisition_id').set_value(
CUtils.dict_value_by_name(metadata_bus_dict, 'acquisition_id',
None))
result = ndi_table.save_data()
return result
def process_mission(self, dataset) -> str:
ds_subpath = dataset.value_by_name(0, 'query_subpath', '')
ds_root_path = dataset.value_by_name(0, 'query_root_path', '')
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_id = dataset.value_by_name(0, 'query_dir_id', '')
owner_obj_id = dataset.value_by_name(0, 'query_dir_parent_objid', '')
parent_id = dataset.value_by_name(0, 'query_dir_parent_id', '')
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure,
'{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo,
ds_retry_times
)
)
self.update_dir_status(ds_id, process_result, self.ProcStatus_Error)
return process_result
if CUtils.equal_ignore_case(ds_subpath, ''):
result = CResult.merge_result(CResult.Success, '根目录[{0}]不支持识别为对象, 当前流程被忽略!')
self.update_dir_status(ds_id, result)
return result
ds_path_full_name = CFile.join_file(ds_root_path, ds_subpath)
CLogger().debug('处理的子目录为: {0}'.format(ds_path_full_name))
try:
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule,
{
'dsdStorageID': ds_storage_id,
'dsdDirectory': ds_subpath
}
)
ds_rule_content = rule_ds.value_by_name(0, 'dsdScanRule', '')
path_obj = CDMPathInfo(self.FileType_Dir, ds_path_full_name, ds_storage_id, ds_id, parent_id, owner_obj_id,
self.get_mission_db_id(), ds_rule_content)
if not path_obj.file_existed:
path_obj.db_update_status_on_path_invalid()
return CResult.merge_result(
CResult.Success,
'目录[{0}]不存在, 在设定状态后, 顺利结束!'.format(ds_path_full_name)
)
else:
path_obj.db_check_and_update_metadata_rule(
CFile.join_file(ds_path_full_name, self.FileName_MetaData_Rule)
)
path_obj.db_path2object()
result = CResult.merge_result(CResult.Success, '目录[{0}]处理顺利完成!'.format(ds_path_full_name))
self.update_dir_status(ds_id, result)
return result
except Exception as error:
result = CResult.merge_result(
CResult.Failure,
'目录[{0}]识别过程出现错误, 详细情况: {1}!'.format(ds_path_full_name, error.__str__())
)
self.update_dir_status(ds_id, result)
return result
def object_copy_stat(self, storage_id, object_id, object_name,
object_relation_name):
try:
ds_object_stat = CFactory().give_me_db(
self.get_mission_db_id()).one_row(
'''
select sum(dodfilesize), max(dodfilemodifytime) from dm2_storage_obj_detail where dodobjectid = :object_id
''', {'object_id': object_id})
object_size = None
object_last_modify_time = None
if not ds_object_stat.is_empty():
object_size = ds_object_stat.value_by_index(0, 0, 0)
object_last_modify_time = ds_object_stat.value_by_index(
0, 1, None)
batch_root_relation_dir = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select dsddirectory
from dm2_storage_directory
where dsdStorageid = :storage_id and position(dsddirectory || '{0}' in :directory) = 1
and dsddirectory <> ''
order by dsddirectory
limit 1
'''.format(CFile.sep()), {
'storage_id': storage_id,
'directory': object_relation_name
}, object_relation_name)
# 更新当前对象的存储大小, 以及最后修改时间
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dso_volumn_now = :object_size, dso_obj_lastmodifytime = :object_last_modify_time
where dsoid = :object_id
''', {
'object_id': object_id,
'object_size': object_size,
'object_last_modify_time': object_last_modify_time
})
count_copy_same_filename_core = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid
where
dm2_storage.dstid is not null
and dm2_storage.dsttype = '{0}'
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{1}'
'''.format(self.Storage_Type_Core, self.FileType_Dir), {
'object_id': object_id,
'object_name': object_name
}, 0) + CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid
left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where
dm2_storage.dstid is not null
and dm2_storage.dsttype = '{0}'
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{1}'
'''.format(self.Storage_Type_Core, self.FileType_File), {
'object_id': object_id,
'object_name': object_name
}, 0)
count_copy_same_filename_and_size_core = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid
where
dm2_storage.dstid is not null
and dm2_storage.dsttype = '{0}'
and dm2_storage_object.dso_volumn_now = :object_size
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{1}'
'''.format(self.Storage_Type_Core, self.FileType_Dir), {
'object_id': object_id,
'object_name': object_name,
'object_size': object_size
}, 0) + CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid
left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where
dm2_storage.dstid is not null
and dm2_storage.dsttype = '{0}'
and dm2_storage_object.dso_volumn_now = :object_size
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{1}'
'''.format(self.Storage_Type_Core, self.FileType_File), {
'object_id': object_id,
'object_name': object_name,
'object_size': object_size
}, 0)
count_copy_same_filename_same_batch = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid
where
dm2_storage.dstid = :storage_id
and position(:directory in dm2_storage_directory.dsddirectory) = 1
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{0}'
'''.format(self.FileType_Dir), {
'storage_id': storage_id,
'object_id': object_id,
'object_name': object_name,
'directory': batch_root_relation_dir
}, 0) + CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid
left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where
dm2_storage.dstid = :storage_id
and position(:directory in dm2_storage_file.dsffilerelationname) = 1
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{0}'
'''.format(self.FileType_File), {
'storage_id': storage_id,
'object_id': object_id,
'object_name': object_name,
'directory': batch_root_relation_dir
}, 0)
count_copy_same_filename_and_size_same_batch = CFactory(
).give_me_db(self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id
left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid
where
dm2_storage.dstid = :storage_id
and position(:directory in dm2_storage_directory.dsddirectory) = 1
and dm2_storage_object.dso_volumn_now = :object_size
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{0}'
'''.format(self.FileType_Dir), {
'storage_id': storage_id,
'object_id': object_id,
'object_name': object_name,
'object_size': object_size,
'directory': object_relation_name
}, 0) + CFactory().give_me_db(
self.get_mission_db_id()).one_value(
'''
select count(dm2_storage_object.dsoid)
from dm2_storage_object
left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid
left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where
dm2_storage.dstid = :storage_id
and position(:directory in dm2_storage_file.dsffilerelationname) = 1
and dm2_storage_object.dso_volumn_now = :object_size
and dm2_storage_object.dsoobjectname = :object_name
and dm2_storage_object.dsoid <> :object_id
and dm2_storage_object.dsodatatype = '{0}'
'''.format(self.FileType_File), {
'storage_id': storage_id,
'object_id': object_id,
'object_name': object_name,
'object_size': object_size,
'directory': object_relation_name
}, 0)
json_text = None
if count_copy_same_filename_and_size_same_batch + \
count_copy_same_filename_and_size_core + \
count_copy_same_filename_same_batch + count_copy_same_filename_core > 0:
json_obj = CJson()
json_obj.load_obj({
self.Storage_Type_Core: {
self.Name_FileName:
count_copy_same_filename_core,
'{0}_{1}'.format(self.Name_FileName, self.Name_Size):
count_copy_same_filename_and_size_core
},
self.Storage_Type_InBound: {
self.Name_FileName:
count_copy_same_filename_same_batch,
'{0}_{1}'.format(self.Name_FileName, self.Name_Size):
count_copy_same_filename_and_size_same_batch
}
})
json_text = json_obj.to_json()
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsocopystat = :copy_stat
where dsoid = :dsoid
''', {
'dsoid': object_id,
'copy_stat': json_text
})
return CResult.merge_result(self.Success, '数据容量统计和数据重复数据分析成功完成! ')
except Exception as error:
return CResult.merge_result(
self.Failure,
'数据容量统计和数据重复数据分析过程出现错误, 详细情况: {0}'.format(error.__str__()))
def process_mission(self, dataset):
dso_id = dataset.value_by_name(0, 'dsoid', '')
dso_data_type = dataset.value_by_name(0, 'dsodatatype', '')
dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '')
dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '')
CLogger().debug('开始处理对象: {0}.{1}.{2}.{3}的元数据'.format(
dso_id, dso_data_type, dso_object_type, dso_object_name))
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(
dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo, ds_retry_times))
self.db_update_object_status(dso_id, process_result,
self.ProcStatus_Error)
return process_result
ds_file_info = self.get_object_info(dso_id, dso_data_type)
if ds_file_info.value_by_name(0, 'query_object_valid',
self.DB_False) == self.DB_False:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsodetailparsestatus = 0
, dsolastmodifytime = now()
, dsodetailparsememo = '文件或目录不存在，元数据无法解析'
where dsoid = :dsoid
''', {'dsoid': dso_id})
return CResult.merge_result(
self.Success, '文件或目录[{0}]不存在，元数据无法解析, 元数据处理正常结束!'.format(
ds_file_info.value_by_name(0, 'query_object_relation_path',
'')))
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID and position(dsddirectory || '{0}' in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule, {
'dsdStorageID':
ds_file_info.value_by_name(0, 'query_object_storage_id', ''),
'dsdDirectory':
ds_file_info.value_by_name(0, 'query_object_relation_path', '')
})
ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '')
file_info_obj = CDMFilePathInfoEx(
dso_data_type,
ds_file_info.value_by_name(0, 'query_object_fullname', ''),
ds_file_info.value_by_name(0, 'query_object_storage_id', ''),
ds_file_info.value_by_name(0, 'query_object_file_id', ''),
ds_file_info.value_by_name(0, 'query_object_file_parent_id', ''),
ds_file_info.value_by_name(0, 'query_object_owner_id', ''),
self.get_mission_db_id(), ds_rule_content)
plugins_obj = CPluginsMng.plugins(file_info_obj, dso_object_type)
if plugins_obj is None:
return CResult.merge_result(
self.Failure, '文件或目录[{0}]的类型插件[{1}]不存在，对象详情无法解析, 处理结束!'.format(
ds_file_info.value_by_name(0, 'query_object_relation_path',
''), dso_object_type))
try:
plugins_obj.classified()
plugins_information = plugins_obj.get_information()
detail_parser = CDetailParserMng.give_me_parser(
CUtils.dict_value_by_name(
plugins_information, plugins_obj.Plugins_Info_DetailEngine,
None), dso_id, dso_object_name, file_info_obj,
plugins_obj.object_detail_file_full_name_list())
process_result = plugins_obj.parser_detail(detail_parser)
if not CResult.result_success(process_result):
self.db_update_object_status(dso_id, process_result)
return process_result
process_result = self.object_copy_stat(
ds_file_info.value_by_name(0, 'query_object_storage_id', ''),
dso_id, dso_object_name,
ds_file_info.value_by_name(0, 'query_object_relation_path',
''))
if CResult.result_success(process_result):
# 更新父对象的容量和最后修改时间
self.__update_object_owner_object_size_and_modifytime(
file_info_obj.owner_obj_id)
result = CResult.merge_result(
self.Success, '文件或目录[{0}]对象详情解析成功结束!'.format(
ds_file_info.value_by_name(
0, 'query_object_relation_path', '')))
self.db_update_object_status(dso_id, result)
return result
else:
self.db_update_object_status(dso_id, process_result)
return process_result
except Exception as error:
result = CResult.merge_result(
self.Failure, '文件或目录[{0}]对象详情解析过程出现错误! 错误原因为: {1}'.format(
ds_file_info.value_by_name(0, 'query_object_relation_path',
''), error.__str__()))
self.db_update_object_status(dso_id, result)
return result
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204.rar'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204.xls'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204_M_rpc.txt'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204_P_rpc.txt'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204F-1.ige'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204F-1.img'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'GF2365199920181204F-1'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204F-1.img.aux.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204F-1.img.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204F-1.rde'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204F-1.rrd'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204G.dbf'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204G.prj'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204G.sbn'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204G.sbx'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204G.shp'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204G.shx'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204M-1.img'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204M-1.img.aux.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204M.XML'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204P.img'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IUnKnown,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204P.XML'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204P-1.img'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204P-1.img.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204RGB.tfw'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204RGB.tif'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204RGB.tif.aux.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204RGB.tif.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204T.XML'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Y.XML'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Y问题说明.docx'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Q.dbf'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Q.prj'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Q.sbn'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Q.sbx'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Q.shp'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'整景纠正带-{0}GF2365199920181204Q.shx'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}]
def process_mission(self, dataset) -> str:
ds_subpath = dataset.value_by_name(0, 'query_subpath', '')
ds_file_name_with_path = dataset.value_by_name(0,
'query_file_full_name',
'')
ds_dir_id = dataset.value_by_name(0, 'query_dir_id', '')
ds_file_id = dataset.value_by_name(0, 'query_file_id', '')
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_owner_obj_id = dataset.value_by_name(0, 'query_dir_parent_objid',
'')
CLogger().debug('处理的文件为: {0}.{1}'.format(ds_file_id,
ds_file_name_with_path))
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(
dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo, ds_retry_times))
self.update_file_status(ds_file_id, process_result,
self.ProcStatus_Error)
return process_result
try:
sql_get_rule = '''
select dsdScanRule
from dm2_storage_directory
where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1
and dsdScanRule is not null
order by dsddirectory desc
limit 1
'''.format(CFile.sep())
rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row(
sql_get_rule, {
'dsdStorageID': ds_storage_id,
'dsdDirectory': ds_subpath
})
ds_rule_content = rule_ds.value_by_name(0, 'dsdScanRule', '')
file_obj = CDMFileInfo(self.FileType_File, ds_file_name_with_path,
ds_storage_id, ds_file_id,
ds_dir_id, ds_owner_obj_id,
self.get_mission_db_id(), ds_rule_content)
if not file_obj.file_existed:
file_obj.db_update_status_on_file_invalid()
return CResult.merge_result(
CResult.Success,
'文件[{0}]不存在, 在设定状态后, 顺利结束!'.format(ds_file_name_with_path))
else:
file_obj.db_file2object()
result = CResult.merge_result(
CResult.Success,
'文件[{0}]的识别过程顺利完成!'.format(ds_file_name_with_path))
self.update_file_status(ds_file_id, result)
return result
except Exception as error:
result = CResult.merge_result(
CResult.Failure,
'文件[{0}]的识别过程出现错误, 详细情况: {1}!'.format(ds_file_name_with_path,
error.__str__()))
self.update_file_status(ds_file_id, result)
return result
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.img'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'632701BJ2+GF1+GJ1+GF2+ZY3DOM'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.img.aux.xml'.format(
CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IUnKnown,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.img.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IUnKnown,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.rrd'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}632701玉树市.mdb'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}63270102XQ.dbf'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}63270102XQ.prj'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}63270102XQ.sbn'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}63270102XQ.sbx'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}63270102XQ.shp'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'非分块{0}63270102XQ.shx'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}]
def test_file_info_list(self):
return [{
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test{0}单景test.tif'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'单景test'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test{0}单景test.tfw'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test{0}单景test.txt'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test{0}单景test_21at.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test_123{0}单景test_123.tif'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown,
self.Name_Test_object_name:
'单景test_123'
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test_123{0}单景test_123.tfw'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test_123{0}单景test_123.txt'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}, {
self.Name_Test_File_Type:
self.FileType_File,
self.Name_Test_file_path:
'单景test_123{0}单景test_123_21at.xml'.format(CFile.sep()),
self.Name_Test_object_confirm:
self.Object_Confirm_IKnown_Not,
self.Name_Test_object_name:
None
}]