def test_file_info_list(self): return [ { self.Name_Test_File_Type: self.FileType_Dir, self.Name_Test_file_path: 'gdb数据集{0}FileGeodb.gdb'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'FileGeodb' }, { self.Name_Test_File_Type: self.FileType_Dir, self.Name_Test_file_path: 'gdb数据集{0}FileGeodb2.gdb'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'FileGeodb2' } # , # { # self.Name_Test_File_Type: self.FileType_Dir, # self.Name_Test_file_path: 'gdb数据集{0}FileGeoNone.gdb'.format(CFile.sep()), # self.Name_Test_object_confirm: self.Object_Confirm_IKnown, # self.Name_Test_object_name: 'FileGeoNone' # } ]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828F.img' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'BJ21_002E74VI_001_0120200730014001_00520200828F' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828M.img' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828P.img' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828T.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}ZhengJing{0}1990天津任意直角坐标系{0}BJ2{0}BJ2-1_002E74VI_001_0120200730014001_005-20200828{0}BJ21_002E74VI_001_0120200730014001_00520200828Y.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_Dir, self.Name_Test_file_path: '202008{0}参考dom'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '参考dom' }]
def process_mission(self, dataset) -> str: ds_id = dataset.value_by_name(0, 'query_dir_id', '') ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') inbound_id = dataset.value_by_name(0, 'query_dir_ib_id', None) ds_subpath = dataset.value_by_name(0, 'query_subpath', '') ds_root_path = dataset.value_by_name(0, 'query_rootpath', '') ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str( dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times)) self.update_dir_status(ds_id, process_result, self.ProcStatus_Error) return process_result if ds_subpath == '': ds_subpath = ds_root_path else: ds_subpath = CFile.join_file(ds_root_path, ds_subpath) # 将所有子目录, 文件的可用性, 都改为未知!!!! self.init_file_or_subpath_valid_unknown(ds_id) try: sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and position((dsddirectory || '{0}') in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_storage_id, 'dsdDirectory': ds_subpath }) ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '') CLogger().debug('处理的目录为: {0}'.format(ds_subpath)) self.parser_file_or_subpath_of_path(dataset, ds_id, ds_subpath, ds_rule_content, inbound_id) result = CResult.merge_result( self.Success, '目录为[{0}]下的文件和子目录扫描处理成功!'.format(ds_subpath)) self.update_dir_status(ds_id, result) return result except Exception as err: result = CResult.merge_result( self.Failure, '目录为[{0}]下的文件和子目录扫描处理出现错误!错误原因为: {1}'.format( ds_subpath, err.__str__())) self.update_dir_status(ds_id, result) return result finally: self.exchange_file_or_subpath_valid_unknown2invalid(ds_id)
def test_file_info_list(self): return [ { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '北京市20200101自由分幅测试{0}影像{0}b.tif'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'b' } ]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020A.TIF' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'J50E005015BJ208M2020A' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020A.TWF' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020P.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000国家标准坐标系{0}ZY320M{0}J50{0}J50E003015{0}J50E005015BJ208M2020M.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }]
def test_file_info_list(self): return [ { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}镶嵌影像成果{0}区域镶嵌{0}1990天津任意直角坐标系{0}tianjin_120116滨海新区_202008_Bj2_08m_tj1990.img'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'tianjin_120116滨海新区_202008_Bj2_08m_tj1990' } ]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}镶嵌影像成果{0}区域镶嵌{0}其他国家标准坐标系{0}a.img'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'a' }]
def process(self) -> str: """ 完成 负责人 张源博、赵宇飞 在这里提取影像数据的快视图, 将元数据文件存储在self.file_content.view_root_dir下 注意返回的串中有快视图和拇指图的文件名 注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!! :return: """ # 获取对象类型 type = 'default' group = 'default' catalog = 'default' # 构建数据对象object对应的识别插件,获取get_information里面的信息 class_classified_obj = CObject.get_plugins_instance_by_object_id(self.file_info.db_server_id, self.object_id) if class_classified_obj is not None: plugins_info = class_classified_obj.get_information() type = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Type, 'default') group = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Group, 'default') catalog = CUtils.dict_value_by_name(plugins_info, class_classified_obj.Plugins_Info_Catalog, 'default') create_time = CTime.today() create_format_time = CTime.format_str(create_time, '%Y%m%d') year = CTime.format_str(create_time, '%Y') month = CTime.format_str(create_time, '%m') day = CTime.format_str(create_time, '%d') sep = CFile.sep() # 操作系统的不同处理分隔符不同 sep_list = [catalog, group, type, year, month, day] relative_path_part = sep.join(sep_list) # 相对路径格式 view_relative_path_browse = r'{2}{0}{2}{1}_browse.png'.format(relative_path_part, self.object_id, sep) view_relative_path_thumb = r'{2}{0}{2}{1}_thumb.jpg'.format(relative_path_part, self.object_id, sep) view_relative_path_geotiff = r'{2}{0}{2}{1}_browse.tiff'.format(relative_path_part, self.object_id, sep) browse_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_browse) thumb_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_thumb) geotiff_full_path = CFile.join_file(self.file_content.view_root_dir, view_relative_path_geotiff) # 进程调用模式 json_out_view = CJson() json_out_view.set_value_of_name('image_path', self.file_info.file_name_with_full_path) json_out_view.set_value_of_name('browse_full_path', browse_full_path) json_out_view.set_value_of_name('thumb_full_path', thumb_full_path) json_out_view.set_value_of_name('geotiff_full_path', geotiff_full_path) result_view = CProcessUtils.processing_method(self.create_view_json, json_out_view) # result_view = self.create_view(self.file_info.file_name_with_full_path, browse_full_path, thumb_full_path, # geotiff_full_path) # result_view = self.create_view_json(json_out_view) if CResult.result_success(result_view): result = CResult.merge_result(self.Success, '处理完毕!') result = CResult.merge_result_info(result, self.Name_Browse, view_relative_path_browse) result = CResult.merge_result_info(result, self.Name_Thumb, view_relative_path_thumb) result = CResult.merge_result_info(result, self.Name_Browse_GeoTiff, view_relative_path_geotiff) else: result = result_view return result
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_Dir, self.Name_Test_file_path: '202008{0}信息产品{0}2020年土地执法项目{0}矢量成果'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '矢量成果' }]
def db_update_status_on_path_invalid(self): """ 处理目录不存在时的业务 1. 标记已经入库的文件为无效 1. 标记已经入库的子目录为无效 :return: """ path_name_with_relation_path = CFile.join_file( self.file_path_with_rel_path, '') params = dict() params['dsdStorageID'] = self.storage_id params['dsdSubDirectory'] = path_name_with_relation_path sql_update_file_invalid = ''' update dm2_storage_file set dsffilevalid = 0, dsfscanstatus = 0 where dsfdirectoryid in ( select dsdid from dm2_storage_directory where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1 ) '''.format(CFile.sep()) sql_update_path_invalid = ''' update dm2_storage_directory set dsd_directory_valid = 0, dsdscanstatus = 0, dsdscanfilestatus = 0, dsdscandirstatus = 0 where dsdstorageid = :dsdStorageID and position(:dsdSubDirectory in dsddirectory || '{0}') = 1 '''.format(CFile.sep()) engine = CFactory().give_me_db(self.db_server_id) session = engine.give_me_session() try: engine.session_execute(session, sql_update_file_invalid, params) engine.session_execute(session, sql_update_path_invalid, params) engine.session_commit(session) except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__())) engine.session_rollback(session) finally: engine.session_close(session)
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}其他国家标准坐标系{0}2772.0-509.0.tif'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '2772.0-509.0' }]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}镶嵌影像成果{0}市域镶嵌{0}2000国家标准坐标系{0}tianjin_202008_bj2_08m_cgcs2000.img' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'tianjin_202008_bj2_08m_cgcs2000' }]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '影像{0}云南高分影像{0}F47{0}' 'F47E001007BJ210M2017A.MDB'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'F47E001007BJ210M2017A' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '影像{0}云南高分影像{0}F48{0}' 'F48E001001BJ210M2017A.MDB'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'F48E001001BJ210M2017A' }]
def test_file_info_list(self): return [ { self.Name_Test_File_Type: self.FileType_Dir, self.Name_Test_file_path: 'mbtiles{0}hdqimage201510_0.mbtiles'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'hdqimage201510_0' }, { self.Name_Test_File_Type: self.FileType_Dir, self.Name_Test_file_path: 'mbtiles{0}hdqimage201510.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None } ]
def test_file_info_list(self): return [ { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018M.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018O.tfw'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018O.tif'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'G49G001038GF210M2018O' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '分幅{0}G49G001038GF210M2018P.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None } ]
def get_custom_affiliated_file_character(self): file_path = self.file_info.file_path file_main_name = self.file_info.file_main_name regularexpression = '(?i)^' + file_main_name[:-1] + r'.\..*' letter_location_FenFu = file_path.find('FenFu') file_path_sub = file_path[letter_location_FenFu + 6:] letter_location = file_path_sub.find(CFile.sep()) shp_path = CFile.join_file(file_path[:letter_location_FenFu + 5], file_path_sub[:letter_location]) return [ { self.Name_FilePath: file_path, # 附属文件的路径 self.Name_RegularExpression: regularexpression, # 附属文件的匹配规则 # 应该从上面匹配到的文件剔除的文件的匹配规则 self.Name_No_Match_RegularExpression: '(?i)^' + file_main_name + r'\..*$' }, { self.Name_FilePath: shp_path, # 附属文件的路径 self.Name_RegularExpression: r'(?i)\.shp$', # 附属文件的匹配规则 # 应该从上面匹配到的文件剔除的文件的匹配规则 self.Name_No_Match_RegularExpression: None } ]
def process_mission(self, dataset): dso_id = dataset.value_by_name(0, 'dsoid', '') dso_data_type = dataset.value_by_name(0, 'dsodatatype', '') dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '') dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '') CLogger().debug('开始处理对象: {0}.{1}.{2}.{3}的元数据'.format( dso_id, dso_data_type, dso_object_type, dso_object_name)) dso_object_retry_times = dataset.value_by_name(0, 'retry_times', 0) if dso_object_retry_times >= self.abnormal_job_retry_times(): dso_object_last_process_memo = CUtils.any_2_str( dataset.value_by_name(0, 'dsometadataparsememo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( dso_object_last_process_memo, dso_object_retry_times)) self.db_update_object_exception(dso_id, process_result, self.ProcStatus_Error) return process_result ds_file_info = self.get_object_info(dso_id, dso_data_type) if ds_file_info.value_by_name(0, 'query_object_valid', self.DB_False) == self.DB_False: process_result = CResult.merge_result( self.Success, '文件或目录[{0}]不存在,元数据无法解析, 元数据处理正常结束!'.format( ds_file_info.value_by_name(0, 'query_object_fullname', ''))) self.db_update_object_status(dso_id, process_result) return process_result sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_file_info.value_by_name(0, 'query_object_storage_id', ''), 'dsdDirectory': ds_file_info.value_by_name(0, 'query_object_relation_path', '') }) ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '') file_info_obj = CDMFilePathInfoEx( dso_data_type, ds_file_info.value_by_name(0, 'query_object_fullname', ''), ds_file_info.value_by_name(0, 'query_object_storage_id', ''), ds_file_info.value_by_name(0, 'query_object_file_id', ''), ds_file_info.value_by_name(0, 'query_object_file_parent_id', ''), ds_file_info.value_by_name(0, 'query_object_owner_id', ''), self.get_mission_db_id(), ds_rule_content) plugins_obj = CPluginsMng.plugins(file_info_obj, dso_object_type) if plugins_obj is None: process_result = CResult.merge_result( self.Failure, '文件或目录[{0}]的类型插件[{1}]不存在,元数据无法解析, 处理结束!'.format( ds_file_info.value_by_name(0, 'query_object_fullname', ''), dso_object_type)) self.db_update_object_status(dso_id, process_result, self.ProcStatus_Error) return process_result plugins_obj.classified() if not plugins_obj.create_virtual_content(): process_result = CResult.merge_result( self.Failure, '文件或目录[{0}]的内容提取失败, 元数据无法提取!'.format( ds_file_info.value_by_name(0, 'query_object_fullname', ''))) self.db_update_object_status(dso_id, process_result) return process_result try: metadata_parser = CMetaDataParser(dso_id, dso_object_name, file_info_obj, plugins_obj.file_content, plugins_obj.get_information()) process_result = plugins_obj.parser_metadata(metadata_parser) if CResult.result_success(process_result): message = '' step_success = ( metadata_parser.metadata.metadata_extract_result != self.DB_False) if not step_success: message = CUtils.str_append(message, '实体元数据解析出现错误', ', ') all_step_success = step_success step_success = metadata_parser.metadata.metadata_bus_extract_result != self.DB_False if not step_success: message = CUtils.str_append(message, '实体元数据解析出现错误', ', ') all_step_success = all_step_success and step_success step_success = metadata_parser.metadata.metadata_view_extract_result != self.DB_False if not step_success: message = CUtils.str_append(message, '快视图等可视化元数据解析出现错误', ', ') all_step_success = all_step_success and step_success step_success = metadata_parser.metadata.metadata_time_extract_result != self.DB_False if not step_success: message = CUtils.str_append(message, '时间元数据解析出现错误', ', ') all_step_success = all_step_success and step_success step_success = metadata_parser.metadata.metadata_spatial_extract_result != self.DB_False if not step_success: message = CUtils.str_append(message, '空间投影元数据解析出现错误', ', ') all_step_success = all_step_success and step_success if all_step_success: self.db_update_object_status(dso_id, process_result) return process_result else: process_result = CResult.merge_result( self.Failure, message) self.db_update_object_status(dso_id, process_result) return process_result else: self.db_update_object_status(dso_id, process_result) return process_result except Exception as error: process_result = CResult.merge_result( self.Failure, '文件或目录[{0}]元数据解析过程出现异常! 错误原因为: {1}'.format( ds_file_info.value_by_name(0, 'query_object_fullname', ''), error.__str__())) self.db_update_object_exception(dso_id, process_result) return process_result finally: plugins_obj.destroy_virtual_content()
def process_mission(self, dataset): dso_id = dataset.value_by_name(0, 'dsoid', '') dso_data_type = dataset.value_by_name(0, 'dsodatatype', '') dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '') dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '') CLogger().debug('开始处理对象: {0}.{1}.{2}.{3}的元数据'.format( dso_id, dso_data_type, dso_object_type, dso_object_name)) ds_object_info = self.get_object_info(dso_id, dso_data_type) ds_object_storage_option = ds_object_info.value_by_name( 0, 'query_object_storage_option', None) if ds_object_info.value_by_name(0, 'query_object_valid', self.DB_False) == self.DB_False: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsotagsparsestatus = 0 , dsolastmodifytime = now() , dsotagsparsememo = '文件或目录不存在,标签无法解析' where dsoid = :dsoid ''', {'dsoid': dso_id}) return CResult.merge_result( self.Success, '文件或目录[{0}]不存在,标签处理正常结束!'.format( ds_object_info.value_by_name(0, 'query_object_fullname', ''))) sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_object_info.value_by_name(0, 'query_object_storage_id', ''), 'dsdDirectory': ds_object_info.value_by_name(0, 'query_object_relation_path', '') }) ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '') file_info_obj = CDMFilePathInfoEx( dso_data_type, ds_object_info.value_by_name(0, 'query_object_fullname', ''), ds_object_info.value_by_name(0, 'query_object_storage_id', ''), ds_object_info.value_by_name(0, 'query_object_file_id', ''), ds_object_info.value_by_name(0, 'query_object_file_parent_id', ''), ds_object_info.value_by_name(0, 'query_object_owner_id', ''), self.get_mission_db_id(), ds_rule_content) plugins_obj = CPluginsMng.plugins(file_info_obj, dso_object_type) if plugins_obj is None: return CResult.merge_result( self.Failure, '文件或目录[{0}]的类型插件[{1}]不存在,对象详情无法解析, 处理结束!'.format( ds_object_info.value_by_name(0, 'query_object_fullname', ''), dso_object_type)) try: plugins_information = plugins_obj.get_information() tags_parser_rule = CUtils.dict_value_by_name( plugins_information, plugins_obj.Plugins_Info_TagsEngine, None) if tags_parser_rule is None: tags_parser_rule = CJson.json_attr_value( CUtils.any_2_str(ds_object_storage_option), self.Path_Setting_MetaData_Tags_Rule, None) if tags_parser_rule is None: tags_parser_rule = settings.application.xpath_one( self.Path_Setting_MetaData_Tags_Rule, None) if tags_parser_rule is None: process_result = CResult.merge_result( CResult.Success, '系统未设置标签库和识别模式, 标签解析将自动结束') else: process_result = plugins_obj.parser_tags( CTagsParser(dso_id, dso_object_name, file_info_obj, dataset.value_by_name(0, 'dsoaliasname', ''), tags_parser_rule)) self.db_update_object_status(dso_id, process_result) return process_result except Exception as err: process_result = CResult.merge_result( self.Failure, '文件或目录[{0}]对象业务分类解析过程出现错误! 错误原因为: {1}'.format( ds_object_info.value_by_name(0, 'query_object_fullname', ''), err.__str__())) self.db_update_object_status(dso_id, process_result) return process_result
def process_main_table(self): object_table_id = self._obj_id # 获取oid object_table_data = self._dataset metadata_bus_dict = self.get_metadata_bus_dict() main_table_name = CUtils.dict_value_by_name(self.information(), 'main_table_name', 'ap_product') main_table = CTable() main_table.load_info(self._db_id, main_table_name) main_table.column_list.column_by_name('id').set_value(object_table_id) productname = CUtils.dict_value_by_name(metadata_bus_dict, 'productname', None) if CUtils.equal_ignore_case(productname, ''): productname = object_table_data.value_by_name( 0, 'dsoobjectname', None) main_table.column_list.column_by_name('productname').set_value( productname) main_table.column_list.column_by_name('producttype').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'producttype', None)) main_table.column_list.column_by_name('regioncode').set_null() main_table.column_list.column_by_name('productattribute').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'productattribute', None)) centerlatitude = CUtils.dict_value_by_name(metadata_bus_dict, 'centerlatitude', None) centerlongitude = CUtils.dict_value_by_name(metadata_bus_dict, 'centerlongitude', None) centerlonlat = '{0},{1}'.format(centerlongitude, centerlatitude) main_table.column_list.column_by_name('centerlonlat').set_value( centerlonlat) main_table.column_list.column_by_name('geomwkt').set_sql(''' st_astext( (select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}') ) '''.format(object_table_id)) main_table.column_list.column_by_name('geomobj').set_sql(''' (select dso_geo_wgs84 from dm2_storage_object where dsoid='{0}') '''.format(object_table_id)) main_table.column_list.column_by_name('browserimg').set_value( '{0}{1}'.format( CFile.sep(), object_table_data.value_by_name(0, 'dso_browser', None))) main_table.column_list.column_by_name('thumbimg').set_value( '{0}{1}'.format( CFile.sep(), object_table_data.value_by_name(0, 'dso_thumb', None))) main_table.column_list.column_by_name('publishdate').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'publishdate', None)) main_table.column_list.column_by_name('copyright').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'copyright', None)) dso_time = object_table_data.value_by_name(0, 'dso_time', None) dso_time_json = CJson() dso_time_json.load_obj(dso_time) imgdate = dso_time_json.xpath_one('time', None) if not CUtils.equal_ignore_case(imgdate, ''): main_table.column_list.column_by_name('imgdate').set_value( imgdate[0:4]) else: main_table.column_list.column_by_name('imgdate').set_null() main_table.column_list.column_by_name('starttime').set_value( dso_time_json.xpath_one('start_time', None)) main_table.column_list.column_by_name('endtime').set_value( dso_time_json.xpath_one('end_time', None)) resolution = CUtils.any_2_str( CUtils.dict_value_by_name(metadata_bus_dict, 'resolution', None)) if not CUtils.equal_ignore_case(resolution, ''): if '/' in resolution: resolution_list = resolution.split('/') temp_list = list() for resolution in resolution_list: temp_list.append(CUtils.to_decimal(resolution, -1)) temp_list = list(set(temp_list)) # 去重 if -1 in temp_list: temp_list.remove(-1) if len(temp_list) > 0: main_table.column_list.column_by_name( 'resolution').set_value(min(temp_list)) else: main_table.column_list.column_by_name( 'resolution').set_value(0) else: main_table.column_list.column_by_name('resolution').set_value( resolution) else: main_table.column_list.column_by_name('resolution').set_value(0) main_table.column_list.column_by_name('filesize').set_sql(''' (select sum(dodfilesize) from dm2_storage_obj_detail where dodobjectid='{0}') '''.format(object_table_id)) productid = CUtils.dict_value_by_name(metadata_bus_dict, 'productid', None) if CUtils.equal_ignore_case(productid, ''): object_type = object_table_data.value_by_name(0, 'dsodatatype', '') if CUtils.equal_ignore_case(object_type, self.Name_Dir): main_table.column_list.column_by_name('productid').set_value( object_table_data.value_by_name(0, 'dsoobjectname', None)) elif CUtils.equal_ignore_case(object_type, self.Name_File): main_table.column_list.column_by_name('productid').set_sql(''' (SELECT dsffilename FROM dm2_storage_file WHERE dsf_object_id = '{0}') '''.format(object_table_id)) else: main_table.column_list.column_by_name('productid').set_null() else: main_table.column_list.column_by_name('productid').set_value( productid) main_table.column_list.column_by_name('remark').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'remark', None)) main_table.column_list.column_by_name('extent').set_sql(''' (select dso_geo_bb_wgs84 from dm2_storage_object where dsoid='{0}') '''.format(object_table_id)) main_table.column_list.column_by_name('proj').set_null() # 原始数据保持空 main_table.column_list.column_by_name('dataid').set_value( object_table_id) main_table.column_list.column_by_name('shplog').set_null() if not main_table.if_exists(): now_time = CUtils.any_2_str( datetime.datetime.now().strftime('%F %T')) main_table.column_list.column_by_name('addtime').set_value( now_time) main_table.column_list.column_by_name('isdel').set_value(0) main_table.column_list.column_by_name('projectnames').set_value( 'productname') result = main_table.save_data() return result
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020O.tif' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '272-500BJ208M2020O' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020O.TFW ' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020P.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-500{0}272-500BJ208M2020M.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020a.tif' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '272-501BJ208M2020a' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020B.tif' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020C.tif' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020D.tif' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020A.TFW ' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020P.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '202008{0}FenFu{0}2000天津城市坐标系{0}BJ208M{0}272-501{0}272-501BJ208M2020M.XML' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.dbf'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.prj'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.sbn'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.sbx'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.shp'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.shp.xml' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IUnKnown, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_1_FBZXQK.shx'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.dbf'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.prj'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.sbn'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.sbx'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.shp'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.shp.xml' .format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IUnKnown, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}正确数据{0}' 'BJ2_GF1_GF2_SPOT_KUNMING_20170204_20171116_CGCS2000_6_FBZXQK.shx'. format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.dbf'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.prj'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.sbn'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.shp'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.sbx'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'BJ2_PL_昌平区_20190608_20190614_WGS84_FBZXQK.shx'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.dbf'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.prj'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.sbn'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.sbx'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.shp'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.shx'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '矢量数据{0}错误数据{0}' 'qb_wv_kunming_20080318_20121214_cgcs2000_FBZXQK.shp.xml'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IUnKnown, self.Name_Test_object_name: None }]
def process_ndi_table(self): object_table_id = self._obj_id # 获取oid metadata_bus_dict = self.get_metadata_bus_dict() ndi_table_name = CUtils.dict_value_by_name(self.information(), 'ndi_table_name', 'ap_product_ndi') ndi_table = CTable() ndi_table.load_info(self._db_id, ndi_table_name) ndi_table.column_list.column_by_name('id').set_value(object_table_id) productname = CUtils.dict_value_by_name(metadata_bus_dict, 'productname', None) if CUtils.equal_ignore_case(productname, ''): productname = self._dataset.value_by_name(0, 'dsoobjectname', None) ndi_table.column_list.column_by_name('rid').set_value(productname) ndi_table.column_list.column_by_name('fid').set_value(object_table_id) ndi_table.column_list.column_by_name('satelliteid').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'satelliteid', None)) ndi_table.column_list.column_by_name('sensorid').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'sensorid', None)) ndi_table.column_list.column_by_name('topleftlatitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'topleftlatitude', None)) ndi_table.column_list.column_by_name('topleftlongitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'topleftlongitude', None)) ndi_table.column_list.column_by_name('toprightlatitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'toprightlatitude', None)) ndi_table.column_list.column_by_name('toprightlongitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'toprightlongitude', None)) ndi_table.column_list.column_by_name('bottomrightlatitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'bottomrightlatitude', None)) ndi_table.column_list.column_by_name('bottomrightlongitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'bottomrightlongitude', None)) ndi_table.column_list.column_by_name('bottomleftlatitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'bottomleftlatitude', None)) ndi_table.column_list.column_by_name('bottomleftlongitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'bottomleftlongitude', None)) ndi_table.column_list.column_by_name('centerlatitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'centerlatitude', None)) ndi_table.column_list.column_by_name('centerlongitude').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'centerlongitude', None)) transformimg = CUtils.dict_value_by_name(metadata_bus_dict, 'transformimg', None) if not CUtils.equal_ignore_case(transformimg, ''): view_path = settings.application.xpath_one( self.Path_Setting_MetaData_Dir_View, None) browser_path = CFile.file_path( self._dataset.value_by_name(0, 'dso_browser', None)) file_list = CFile.file_or_dir_fullname_of_path( CFile.join_file(view_path, browser_path), False, transformimg, CFile.MatchType_Regex) if len(file_list) > 0: ndi_table.column_list.column_by_name('transformimg').set_value( '{0}{1}'.format( CFile.sep(), CFile.join_file(browser_path, CFile.file_name(file_list[0])))) ndi_table.column_list.column_by_name('filesize').set_sql(''' (select sum(dodfilesize) from dm2_storage_obj_detail where dodobjectid='{0}') '''.format(object_table_id)) ndi_table.column_list.column_by_name('dataexist').set_value(0) ndi_table.column_list.column_by_name('centertime').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'centertime', None)) resolution = CUtils.dict_value_by_name(metadata_bus_dict, 'resolution', None) if not CUtils.equal_ignore_case(resolution, ''): ndi_table.column_list.column_by_name('resolution').set_value( resolution) else: ndi_table.column_list.column_by_name('resolution').set_value(0) rollangle = CUtils.dict_value_by_name(metadata_bus_dict, 'rollangle', 0) if CUtils.equal_ignore_case(rollangle, ''): rollangle = 0 ndi_table.column_list.column_by_name('rollangle').set_value(rollangle) cloudpercent = CUtils.dict_value_by_name(metadata_bus_dict, 'cloudpercent', 0) if CUtils.equal_ignore_case(cloudpercent, ''): cloudpercent = 0 ndi_table.column_list.column_by_name('cloudpercent').set_value( cloudpercent) ndi_table.column_list.column_by_name('dataum').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'dataum', None)) ndi_table.column_list.column_by_name('acquisition_id').set_value( CUtils.dict_value_by_name(metadata_bus_dict, 'acquisition_id', None)) result = ndi_table.save_data() return result
def process_mission(self, dataset) -> str: ds_subpath = dataset.value_by_name(0, 'query_subpath', '') ds_root_path = dataset.value_by_name(0, 'query_root_path', '') ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_id = dataset.value_by_name(0, 'query_dir_id', '') owner_obj_id = dataset.value_by_name(0, 'query_dir_parent_objid', '') parent_id = dataset.value_by_name(0, 'query_dir_parent_id', '') ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str(dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times ) ) self.update_dir_status(ds_id, process_result, self.ProcStatus_Error) return process_result if CUtils.equal_ignore_case(ds_subpath, ''): result = CResult.merge_result(CResult.Success, '根目录[{0}]不支持识别为对象, 当前流程被忽略!') self.update_dir_status(ds_id, result) return result ds_path_full_name = CFile.join_file(ds_root_path, ds_subpath) CLogger().debug('处理的子目录为: {0}'.format(ds_path_full_name)) try: sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_storage_id, 'dsdDirectory': ds_subpath } ) ds_rule_content = rule_ds.value_by_name(0, 'dsdScanRule', '') path_obj = CDMPathInfo(self.FileType_Dir, ds_path_full_name, ds_storage_id, ds_id, parent_id, owner_obj_id, self.get_mission_db_id(), ds_rule_content) if not path_obj.file_existed: path_obj.db_update_status_on_path_invalid() return CResult.merge_result( CResult.Success, '目录[{0}]不存在, 在设定状态后, 顺利结束!'.format(ds_path_full_name) ) else: path_obj.db_check_and_update_metadata_rule( CFile.join_file(ds_path_full_name, self.FileName_MetaData_Rule) ) path_obj.db_path2object() result = CResult.merge_result(CResult.Success, '目录[{0}]处理顺利完成!'.format(ds_path_full_name)) self.update_dir_status(ds_id, result) return result except Exception as error: result = CResult.merge_result( CResult.Failure, '目录[{0}]识别过程出现错误, 详细情况: {1}!'.format(ds_path_full_name, error.__str__()) ) self.update_dir_status(ds_id, result) return result
def object_copy_stat(self, storage_id, object_id, object_name, object_relation_name): try: ds_object_stat = CFactory().give_me_db( self.get_mission_db_id()).one_row( ''' select sum(dodfilesize), max(dodfilemodifytime) from dm2_storage_obj_detail where dodobjectid = :object_id ''', {'object_id': object_id}) object_size = None object_last_modify_time = None if not ds_object_stat.is_empty(): object_size = ds_object_stat.value_by_index(0, 0, 0) object_last_modify_time = ds_object_stat.value_by_index( 0, 1, None) batch_root_relation_dir = CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select dsddirectory from dm2_storage_directory where dsdStorageid = :storage_id and position(dsddirectory || '{0}' in :directory) = 1 and dsddirectory <> '' order by dsddirectory limit 1 '''.format(CFile.sep()), { 'storage_id': storage_id, 'directory': object_relation_name }, object_relation_name) # 更新当前对象的存储大小, 以及最后修改时间 CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dso_volumn_now = :object_size, dso_obj_lastmodifytime = :object_last_modify_time where dsoid = :object_id ''', { 'object_id': object_id, 'object_size': object_size, 'object_last_modify_time': object_last_modify_time }) count_copy_same_filename_core = CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid where dm2_storage.dstid is not null and dm2_storage.dsttype = '{0}' and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{1}' '''.format(self.Storage_Type_Core, self.FileType_Dir), { 'object_id': object_id, 'object_name': object_name }, 0) + CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid where dm2_storage.dstid is not null and dm2_storage.dsttype = '{0}' and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{1}' '''.format(self.Storage_Type_Core, self.FileType_File), { 'object_id': object_id, 'object_name': object_name }, 0) count_copy_same_filename_and_size_core = CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid where dm2_storage.dstid is not null and dm2_storage.dsttype = '{0}' and dm2_storage_object.dso_volumn_now = :object_size and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{1}' '''.format(self.Storage_Type_Core, self.FileType_Dir), { 'object_id': object_id, 'object_name': object_name, 'object_size': object_size }, 0) + CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid where dm2_storage.dstid is not null and dm2_storage.dsttype = '{0}' and dm2_storage_object.dso_volumn_now = :object_size and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{1}' '''.format(self.Storage_Type_Core, self.FileType_File), { 'object_id': object_id, 'object_name': object_name, 'object_size': object_size }, 0) count_copy_same_filename_same_batch = CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid where dm2_storage.dstid = :storage_id and position(:directory in dm2_storage_directory.dsddirectory) = 1 and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{0}' '''.format(self.FileType_Dir), { 'storage_id': storage_id, 'object_id': object_id, 'object_name': object_name, 'directory': batch_root_relation_dir }, 0) + CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid where dm2_storage.dstid = :storage_id and position(:directory in dm2_storage_file.dsffilerelationname) = 1 and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{0}' '''.format(self.FileType_File), { 'storage_id': storage_id, 'object_id': object_id, 'object_name': object_name, 'directory': batch_root_relation_dir }, 0) count_copy_same_filename_and_size_same_batch = CFactory( ).give_me_db(self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_directory on dm2_storage_object.dsoid = dm2_storage_directory.dsd_object_id left join dm2_storage on dm2_storage_directory.dsdstorageid = dm2_storage.dstid where dm2_storage.dstid = :storage_id and position(:directory in dm2_storage_directory.dsddirectory) = 1 and dm2_storage_object.dso_volumn_now = :object_size and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{0}' '''.format(self.FileType_Dir), { 'storage_id': storage_id, 'object_id': object_id, 'object_name': object_name, 'object_size': object_size, 'directory': object_relation_name }, 0) + CFactory().give_me_db( self.get_mission_db_id()).one_value( ''' select count(dm2_storage_object.dsoid) from dm2_storage_object left join dm2_storage_file on dm2_storage_file.dsf_object_id = dm2_storage_object.dsoid left join dm2_storage on dm2_storage_file.dsfstorageid = dm2_storage.dstid where dm2_storage.dstid = :storage_id and position(:directory in dm2_storage_file.dsffilerelationname) = 1 and dm2_storage_object.dso_volumn_now = :object_size and dm2_storage_object.dsoobjectname = :object_name and dm2_storage_object.dsoid <> :object_id and dm2_storage_object.dsodatatype = '{0}' '''.format(self.FileType_File), { 'storage_id': storage_id, 'object_id': object_id, 'object_name': object_name, 'object_size': object_size, 'directory': object_relation_name }, 0) json_text = None if count_copy_same_filename_and_size_same_batch + \ count_copy_same_filename_and_size_core + \ count_copy_same_filename_same_batch + count_copy_same_filename_core > 0: json_obj = CJson() json_obj.load_obj({ self.Storage_Type_Core: { self.Name_FileName: count_copy_same_filename_core, '{0}_{1}'.format(self.Name_FileName, self.Name_Size): count_copy_same_filename_and_size_core }, self.Storage_Type_InBound: { self.Name_FileName: count_copy_same_filename_same_batch, '{0}_{1}'.format(self.Name_FileName, self.Name_Size): count_copy_same_filename_and_size_same_batch } }) json_text = json_obj.to_json() CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsocopystat = :copy_stat where dsoid = :dsoid ''', { 'dsoid': object_id, 'copy_stat': json_text }) return CResult.merge_result(self.Success, '数据容量统计和数据重复数据分析成功完成! ') except Exception as error: return CResult.merge_result( self.Failure, '数据容量统计和数据重复数据分析过程出现错误, 详细情况: {0}'.format(error.__str__()))
def process_mission(self, dataset): dso_id = dataset.value_by_name(0, 'dsoid', '') dso_data_type = dataset.value_by_name(0, 'dsodatatype', '') dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '') dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '') CLogger().debug('开始处理对象: {0}.{1}.{2}.{3}的元数据'.format( dso_id, dso_data_type, dso_object_type, dso_object_name)) ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str( dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times)) self.db_update_object_status(dso_id, process_result, self.ProcStatus_Error) return process_result ds_file_info = self.get_object_info(dso_id, dso_data_type) if ds_file_info.value_by_name(0, 'query_object_valid', self.DB_False) == self.DB_False: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsodetailparsestatus = 0 , dsolastmodifytime = now() , dsodetailparsememo = '文件或目录不存在,元数据无法解析' where dsoid = :dsoid ''', {'dsoid': dso_id}) return CResult.merge_result( self.Success, '文件或目录[{0}]不存在,元数据无法解析, 元数据处理正常结束!'.format( ds_file_info.value_by_name(0, 'query_object_relation_path', ''))) sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and position(dsddirectory || '{0}' in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_file_info.value_by_name(0, 'query_object_storage_id', ''), 'dsdDirectory': ds_file_info.value_by_name(0, 'query_object_relation_path', '') }) ds_rule_content = rule_ds.value_by_name(0, 'dsScanRule', '') file_info_obj = CDMFilePathInfoEx( dso_data_type, ds_file_info.value_by_name(0, 'query_object_fullname', ''), ds_file_info.value_by_name(0, 'query_object_storage_id', ''), ds_file_info.value_by_name(0, 'query_object_file_id', ''), ds_file_info.value_by_name(0, 'query_object_file_parent_id', ''), ds_file_info.value_by_name(0, 'query_object_owner_id', ''), self.get_mission_db_id(), ds_rule_content) plugins_obj = CPluginsMng.plugins(file_info_obj, dso_object_type) if plugins_obj is None: return CResult.merge_result( self.Failure, '文件或目录[{0}]的类型插件[{1}]不存在,对象详情无法解析, 处理结束!'.format( ds_file_info.value_by_name(0, 'query_object_relation_path', ''), dso_object_type)) try: plugins_obj.classified() plugins_information = plugins_obj.get_information() detail_parser = CDetailParserMng.give_me_parser( CUtils.dict_value_by_name( plugins_information, plugins_obj.Plugins_Info_DetailEngine, None), dso_id, dso_object_name, file_info_obj, plugins_obj.object_detail_file_full_name_list()) process_result = plugins_obj.parser_detail(detail_parser) if not CResult.result_success(process_result): self.db_update_object_status(dso_id, process_result) return process_result process_result = self.object_copy_stat( ds_file_info.value_by_name(0, 'query_object_storage_id', ''), dso_id, dso_object_name, ds_file_info.value_by_name(0, 'query_object_relation_path', '')) if CResult.result_success(process_result): # 更新父对象的容量和最后修改时间 self.__update_object_owner_object_size_and_modifytime( file_info_obj.owner_obj_id) result = CResult.merge_result( self.Success, '文件或目录[{0}]对象详情解析成功结束!'.format( ds_file_info.value_by_name( 0, 'query_object_relation_path', ''))) self.db_update_object_status(dso_id, result) return result else: self.db_update_object_status(dso_id, process_result) return process_result except Exception as error: result = CResult.merge_result( self.Failure, '文件或目录[{0}]对象详情解析过程出现错误! 错误原因为: {1}'.format( ds_file_info.value_by_name(0, 'query_object_relation_path', ''), error.__str__())) self.db_update_object_status(dso_id, result) return result
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204.rar'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204.xls'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204_M_rpc.txt'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204_P_rpc.txt'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204F-1.ige'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204F-1.img'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: 'GF2365199920181204F-1' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204F-1.img.aux.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204F-1.img.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204F-1.rde'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204F-1.rrd'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204G.dbf'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204G.prj'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204G.sbn'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204G.sbx'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204G.shp'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204G.shx'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204M-1.img'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204M-1.img.aux.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204M.XML'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204P.img'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IUnKnown, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204P.XML'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204P-1.img'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204P-1.img.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204RGB.tfw'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204RGB.tif'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204RGB.tif.aux.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204RGB.tif.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204T.XML'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Y.XML'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Y问题说明.docx'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Q.dbf'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Q.prj'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Q.sbn'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Q.sbx'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Q.shp'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '整景纠正带-{0}GF2365199920181204Q.shx'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }]
def process_mission(self, dataset) -> str: ds_subpath = dataset.value_by_name(0, 'query_subpath', '') ds_file_name_with_path = dataset.value_by_name(0, 'query_file_full_name', '') ds_dir_id = dataset.value_by_name(0, 'query_dir_id', '') ds_file_id = dataset.value_by_name(0, 'query_file_id', '') ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_owner_obj_id = dataset.value_by_name(0, 'query_dir_parent_objid', '') CLogger().debug('处理的文件为: {0}.{1}'.format(ds_file_id, ds_file_name_with_path)) ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str( dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times)) self.update_file_status(ds_file_id, process_result, self.ProcStatus_Error) return process_result try: sql_get_rule = ''' select dsdScanRule from dm2_storage_directory where dsdStorageid = :dsdStorageID and Position(dsddirectory || '{0}' in :dsdDirectory) = 1 and dsdScanRule is not null order by dsddirectory desc limit 1 '''.format(CFile.sep()) rule_ds = CFactory().give_me_db(self.get_mission_db_id()).one_row( sql_get_rule, { 'dsdStorageID': ds_storage_id, 'dsdDirectory': ds_subpath }) ds_rule_content = rule_ds.value_by_name(0, 'dsdScanRule', '') file_obj = CDMFileInfo(self.FileType_File, ds_file_name_with_path, ds_storage_id, ds_file_id, ds_dir_id, ds_owner_obj_id, self.get_mission_db_id(), ds_rule_content) if not file_obj.file_existed: file_obj.db_update_status_on_file_invalid() return CResult.merge_result( CResult.Success, '文件[{0}]不存在, 在设定状态后, 顺利结束!'.format(ds_file_name_with_path)) else: file_obj.db_file2object() result = CResult.merge_result( CResult.Success, '文件[{0}]的识别过程顺利完成!'.format(ds_file_name_with_path)) self.update_file_status(ds_file_id, result) return result except Exception as error: result = CResult.merge_result( CResult.Failure, '文件[{0}]的识别过程出现错误, 详细情况: {1}!'.format(ds_file_name_with_path, error.__str__())) self.update_file_status(ds_file_id, result) return result
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.img'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '632701BJ2+GF1+GJ1+GF2+ZY3DOM' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.img.aux.xml'.format( CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IUnKnown, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.img.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IUnKnown, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}632701BJ2+GF1+GJ1+GF2+ZY3DOM.rrd'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}632701玉树市.mdb'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}63270102XQ.dbf'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}63270102XQ.prj'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}63270102XQ.sbn'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}63270102XQ.sbx'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}63270102XQ.shp'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '非分块{0}63270102XQ.shx'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }]
def test_file_info_list(self): return [{ self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test{0}单景test.tif'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '单景test' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test{0}单景test.tfw'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test{0}单景test.txt'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test{0}单景test_21at.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test_123{0}单景test_123.tif'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown, self.Name_Test_object_name: '单景test_123' }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test_123{0}单景test_123.tfw'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test_123{0}单景test_123.txt'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }, { self.Name_Test_File_Type: self.FileType_File, self.Name_Test_file_path: '单景test_123{0}单景test_123_21at.xml'.format(CFile.sep()), self.Name_Test_object_confirm: self.Object_Confirm_IKnown_Not, self.Name_Test_object_name: None }]