def db_update_object_status(self, dso_id, process_result): CLogger().debug(CResult.result_message(process_result)) if CResult.result_success(process_result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsotagsparsestatus = {0} , dsolastmodifytime = now() , dsotagsparsememo = :dsotagsparsememo where dsoid = :dsoid '''.format(self.ProcStatus_Finished), { 'dsoid': dso_id, 'dsotagsparsememo': CResult.result_message(process_result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsotagsparsestatus = {0} , dsolastmodifytime = now() , dsotagsparsememo = :dsotagsparsememo where dsoid = :dsoid '''.format(self.ProcStatus_Error), { 'dsoid': dso_id, 'dsotagsparsememo': CResult.result_message(process_result) })
def update_notify_result(self, notify_id, result): if CResult.result_success(result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsi_na_status = {0} , dsi_na_proc_id = null , dsi_na_proc_memo = :notify_message , dsiproctime = now() where dsiid = :notify_id '''.format(self.ProcStatus_WaitConfirm), { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsi_na_status = {0} , dsi_na_proc_id = null , dsi_na_proc_memo = :notify_message , dsiproctime = now() where dsiid = :notify_id '''.format(self.ProcStatus_Error), { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) })
def db_update_object_exception(self, dso_id, process_result, process_status=None): if process_status is not None: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsometadataparsestatus = :status , dsometadataparsememo = :dsometadataparsememo , dsolastmodifytime = now() where dsoid = :dsoid '''.format(self.ProcStatus_InQueue), { 'dsoid': dso_id, 'status': process_status, 'dsometadataparsememo': CResult.result_message(process_result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsometadataparsestatus = (dsometadataparsestatus / 10 + 1) * 10 + {0} , dsometadataparsememo = :dsometadataparsememo , dsolastmodifytime = now() where dsoid = :dsoid '''.format(self.ProcStatus_InQueue), { 'dsoid': dso_id, 'dsometadataparsememo': CResult.result_message(process_result) })
def update_inbound_qi_result(self, notify_id, result, storage_type='mix', storage_option=None, ib_option=None): CLogger().debug(CResult.result_message(result)) if CResult.result_success(result): if CUtils.equal_ignore_case(storage_type, self.Storage_Type_InBound): switch_inbound_after_qi_immediately_status = CUtils.equal_ignore_case( settings.application.xpath_one( self.path_switch( self.Path_Setting_MetaData_QI_Switch, self. Switch_Inbound_After_QI_Immediately_Of_IB_Storage), self.Name_ON), self.Name_ON) else: switch_inbound_after_qi_immediately_status = CUtils.equal_ignore_case( settings.application.xpath_one( self.path_switch( self.Path_Setting_MetaData_QI_Switch, self. Switch_Inbound_After_QI_Immediately_Of_MIX_Storage ), self.Name_OFF), self.Name_ON) if switch_inbound_after_qi_immediately_status: next_status = self.IB_Status_IB_InQueue else: next_status = self.IB_Status_QI_Finished CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsiStatus = {0}, dsiprocmemo = :notify_message, dsiproctime = now() where dsiid = :notify_id '''.format(next_status), { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsiprocmemo = :notify_message, dsiproctime = now() where dsiid = :notify_id ''', { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) })
def sync(self) -> str: try: result = self.process_metadata_bus_dict() if not CResult.result_success(result): return CResult.merge_result( self.Failure, '卫星数据的业务元数据的详细内容解析出错!原因为{0}'.format( CResult.result_message(result))) main_result = self.process_main_table() metadata_result = self.process_metadata_table() ndi_result = self.process_ndi_table() if not CResult.result_success(main_result): return main_result elif not CResult.result_success(metadata_result): return metadata_result elif not CResult.result_success(ndi_result): return ndi_result else: return CResult.merge_result( self.Success, '对象[{0}]的同步成功! '.format(self._obj_name)) except Exception as error: return CResult.merge_result( self.Failure, '数据检索分发模块在进行数据同步时出现错误:同步的对象[{0}]在处理时出现异常, 详细情况: [{1}]!'.format( self._obj_name, error.__str__()))
def update_dir_status(self, dir_id, result, status=None): if status is not None: sql_update_directory_status = ''' update dm2_storage_directory set dsdscanstatus = :status, dsdscanmemo = :memo, dsdlastmodifytime = now() where dsdid = :dsdid ''' elif CResult.result_success(result): sql_update_directory_status = ''' update dm2_storage_directory set dsdscanstatus = {0}, dsdscanmemo = :memo, dsdlastmodifytime = now() where dsdid = :dsdid '''.format(self.ProcStatus_Finished) else: sql_update_directory_status = ''' update dm2_storage_directory set dsdscanstatus = (dsdscanstatus / 10 + 1) * 10 + {0}, dsdscanmemo = :memo, dsdlastmodifytime = now() where dsdid = :dsdid '''.format(self.ProcStatus_InQueue) params = dict() params['dsdid'] = dir_id params['memo'] = CResult.result_message(result) params['status'] = status CFactory().give_me_db(self.get_mission_db_id()).execute(sql_update_directory_status, params)
def process(self) -> str: """ todo 负责人 赵宇飞 在这里提取影像数据的空间信息, 以文件形式存储在self.file_content.work_root_dir下 注意返回的串中有空间信息的文件名 注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!! :return: """ result_process = self.process_raster() if CResult.result_success(result_process): file_path = self.file_content.work_root_dir dict_temp_file_name = { self.Name_Native_Center: '{0}_native_center.wkt'.format(self.object_name), self.Name_Native_BBox: '{0}_native_bbox.wkt'.format(self.object_name), self.Name_Native_Geom: '{0}_native_geom.wkt'.format(self.object_name), self.Name_Wgs84_Center: '{0}_wgs84_center.wkt'.format(self.object_name), self.Name_Wgs84_BBox: '{0}_wgs84_bbox.wkt'.format(self.object_name), self.Name_Wgs84_Geom: '{0}_wgs84_geom.wkt'.format(self.object_name) } dict_temp_prj_name = { self.Name_Prj_Wkt: CResult.result_info(result_process, self.Name_Prj_Wkt, None), self.Name_Prj_Proj4: CResult.result_info(result_process, self.Name_Prj_Proj4, None), self.Name_Prj_Project: CResult.result_info(result_process, self.Name_Prj_Project, None), self.Name_Prj_Coordinate: CResult.result_info(result_process, self.Name_Prj_Coordinate, None), self.Name_Prj_Source: CResult.result_info(result_process, self.Name_Prj_Source, None), self.Name_Prj_Zone: CResult.result_info(result_process, self.Name_Prj_Zone, None), self.Name_Prj_Degree: CResult.result_info(result_process, self.Name_Prj_Degree, None) } result = CResult.merge_result(self.Success, '处理完毕!') for file_type, file_name in dict_temp_file_name.items(): result = CResult.merge_result_info(result, file_type, CFile.join_file(file_path, file_name)) for prj_type, prj_name in dict_temp_prj_name.items(): result = CResult.merge_result_info(result, prj_type, prj_name) else: result = CResult.merge_result(self.Failure, CResult.result_message(result_process)) return result
def __update_layer_update_result(self, deploy_id, result): if CResult.result_success(result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dp_v_qfg_layer set dpStatus = 0, dpprocessresult = :message where dpid = :id ''', {'id': deploy_id, 'message': CResult.result_message(result)} ) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dp_v_qfg_layer set dpStatus = 21, dpprocessresult = :message where dpid = :id ''', {'id': deploy_id, 'message': CResult.result_message(result)} )
def update_inbound_na_progress(self, notify_id, message): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsi_na_proc_memo = :notify_message, dsiproctime = now() where dsiid = :notify_id ''', { 'notify_id': notify_id, 'notify_message': CResult.result_message(message) })
def update_sync_result(self, na_id, result): if CResult.result_success(result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_obj_na set dson_notify_status = {0}, dson_notify_proc_id = null, dson_notify_proc_memo = :proc_message where dsonid = :id '''.format(self.ProcStatus_Finished), { 'id': na_id, 'proc_message': CResult.result_message(result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_obj_na set dson_notify_status = {0}, dson_notify_proc_id = null, dson_notify_proc_memo = :proc_message where dsonid = :id '''.format(self.ProcStatus_Error), { 'id': na_id, 'proc_message': CResult.result_message(result) })
def update_inbound_na_result(self, notify_id, result): CLogger().debug(CResult.result_message(result)) if CResult.result_success(result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsi_na_status = {0}, dsi_na_proc_memo = :notify_message, dsiproctime = now() where dsiid = :notify_id '''.format(self.ProcStatus_Finished), { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsi_na_proc_memo = :notify_message, dsiproctime = now() where dsiid = :notify_id ''', { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) })
def update_inbound_qi_result(self, notify_id, result): CLogger().debug(CResult.result_message(result)) if CResult.result_success(result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsiStatus = {0}, dsiprocmemo = :notify_message where dsiid = :notify_id '''.format(self.IB_Status_QI_Processing), { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsiStatus = {0}, dsiprocmemo = :notify_message where dsiid = :notify_id '''.format(self.IB_Status_QI_Error), { 'notify_id': notify_id, 'notify_message': CResult.result_message(result) })
def update_ib_result(self, ib_id, result): if CResult.result_success(result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsistatus = {0}, dsiprocmemo = :ib_message, dsi_na_status = {1} where dsiid = :ib_id '''.format(self.ProcStatus_Finished, self.ProcStatus_InQueue), { 'ib_id': ib_id, 'ib_message': CResult.result_message(result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_inbound set dsistatus = {0}, dsiprocmemo = :ib_message where dsiid = :ib_id '''.format(self.IB_Status_IB_Error), { 'ib_id': ib_id, 'ib_message': CResult.result_message(result) })
def db_update_object_status(self, dso_id, result, status=None): if status is not None: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsodetailparsestatus = :status , dsolastmodifytime = now() , dsodetailparsememo = :dsodetailparsememo where dsoid = :dsoid ''', { 'dsoid': dso_id, 'dsodetailparsememo': CResult.result_message(result), 'status': status }) elif CResult.result_success(result): CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsodetailparsestatus = {0} , dsolastmodifytime = now() , dsodetailparsememo = :dsodetailparsememo where dsoid = :dsoid '''.format(self.ProcStatus_Finished), { 'dsoid': dso_id, 'dsodetailparsememo': CResult.result_message(result) }) else: CFactory().give_me_db(self.get_mission_db_id()).execute( ''' update dm2_storage_object set dsodetailparsestatus = (dsodetailparsestatus / 10 + 1) * 10 + {0}, , dsolastmodifytime = now() , dsodetailparsememo = :dsodetailparsememo where dsoid = :dsoid '''.format(self.ProcStatus_InQueue), { 'dsoid': dso_id, 'dsodetailparsememo': CResult.result_message(result) })
def process_mission(self, dataset): dso_id = dataset.value_by_name(0, 'dsoid', '') dso_data_type = dataset.value_by_name(0, 'dsodatatype', '') dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '') dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '') dso_object_da_content = CUtils.any_2_str(dataset.value_by_name(0, 'dso_da_result', '')) dso_object_quality = dataset.value_by_name(0, 'dso_quality', '') ds_retry_times = dataset.value_by_name(0, 'retry_times', 0) if ds_retry_times >= self.abnormal_job_retry_times(): ds_last_process_memo = CUtils.any_2_str(dataset.value_by_name(0, 'last_process_memo', None)) process_result = CResult.merge_result( self.Failure, '{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format( ds_last_process_memo, ds_retry_times ) ) self.update_status(dso_id, None, process_result, self.ProcStatus_Error) return process_result dso_quality = CXml() dso_quality.load_xml(dso_object_quality) dso_da_json = CJson() dso_da_json.load_json_text(dso_object_da_content) CLogger().debug( '开始处理对象: {0}.{1}.{2}.{3}对各个子系统的支撑能力'.format(dso_id, dso_data_type, dso_object_type, dso_object_name)) try: modules_root_dir = CSys.get_metadata_data_access_modules_root_dir() modules_file_list = CFile.file_or_subpath_of_path( modules_root_dir, '{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py) ) for file_name_without_path in modules_file_list: file_main_name = CFile.file_main_name(file_name_without_path) # 判断模块的可访问是否已经被人工审批, 如果人工审批, 则这里不再计算和覆盖 module_access = dso_da_json.xpath_one( '{0}.{1}'.format(file_main_name, self.Name_Audit), self.Name_System ) if CUtils.equal_ignore_case(module_access, self.Name_User): continue try: module_obj = CObject.create_module_instance( CSys.get_metadata_data_access_modules_root_name(), file_main_name, self.get_mission_db_id() ) module_title = CUtils.dict_value_by_name(module_obj.information(), self.Name_Title, '') result = CUtils.any_2_str(module_obj.access(dso_id, dso_object_name, dso_data_type, dso_quality) ) if CResult.result_success(result): module_access = CResult.result_info(result, self.Name_Access, self.DataAccess_Forbid) else: CLogger().debug('模块[{0}]解析出现错误, 系统将忽略本模块, 继续处理下一个!'.format(file_main_name)) module_access = self.DataAccess_Unknown module_access_message = CResult.result_message(result) module_obj = {self.Name_Audit: self.Name_System, self.Name_Result: module_access, self.Name_Title: module_title, self.Name_Message: module_access_message} dso_da_json.set_value_of_name(file_main_name, module_obj) except Exception as error: CLogger().debug('模块[{0}]解析出现异常, 原因为[{1}], 请检查!'.format(file_main_name, error.__str__())) module_access = self.DataAccess_Unknown module_access_message = '模块[{0}]解析出现异常, 原因为[{1}], 请检查!'.format(file_main_name, error.__str__()) module_obj = {self.Name_Audit: self.Name_System, self.Name_Result: module_access, self.Name_Title: file_main_name, self.Name_Message: module_access_message} dso_da_json.set_value_of_name(file_main_name, module_obj) process_result = CResult.merge_result( self.Success, '对象[{0}.{1}]访问权限解析成功!'.format(dso_id, dso_object_name) ) self.update_status( dso_id, dso_da_json.to_json(), process_result ) return process_result except Exception as error: process_result = CResult.merge_result( self.Failure, '对象[{0}.{1}]访问权限解析出错, 原因为[{2}]!'.format(dso_id, dso_object_name, error.__str__()) ) self.update_status(dso_id, None, process_result) return process_result
def parser_metadata_custom(self, parser: CMetaDataParser) -> str: """ 自定义的元数据解析, 在所有质检和其他处理之后触发 :param parser: :return: """ meta_data_json = parser.metadata.metadata_json() if meta_data_json is None: return CResult.merge_result( self.Success, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 无法获取JSON格式的元数据! '.format( self.file_info.file_name_with_full_path, ) ) json_data_source = meta_data_json.xpath_one('datasource', None) layer_list = meta_data_json.xpath_one(self.Name_Layers, None) if layer_list is None: return CResult.merge_result( self.Success, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 元数据中无法找到layers节点! '.format( self.file_info.file_name_with_full_path, ) ) mdb_ib_id = CFactory().give_me_db(self.file_info.db_server_id).one_value( ''' select dso_ib_id from dm2_storage_object where dsoid = :object_id ''', { 'object_id': parser.object_id } ) error_message_list = [] table = CTable() table.load_info(self.file_info.db_server_id, self.TableName_DM_Object) for layer in layer_list: layer_name = CUtils.dict_value_by_name(layer, self.Name_Name, '') if CUtils.equal_ignore_case(layer_name, ''): continue layer_alias_name = CUtils.dict_value_by_name(layer, self.Name_Description, layer_name) layer_metadata_json = CJson() layer_metadata_json.set_value_of_name('datasource', json_data_source) layer_metadata_json.set_value_of_name('layer_count', 1) layer_metadata_json.set_value_of_name('layers', [layer]) layer_metadata_text = layer_metadata_json.to_json() try: sql_find_layer_existed = ''' select dsoid as layer_id_existed from dm2_storage_object where upper(dsoobjectname) = upper(:layer_name) and dsoparentobjid = :object_id ''' layer_id_existed = CFactory().give_me_db(self.file_info.db_server_id).one_value( sql_find_layer_existed, { 'layer_name': layer_name, 'object_id': parser.object_id } ) if layer_id_existed is None: layer_id_existed = CUtils.one_id() table.column_list.reset() table.column_list.column_by_name('dsoid').set_value(layer_id_existed) table.column_list.column_by_name('dsoobjectname').set_value(layer_name) table.column_list.column_by_name('dsoobjecttype').set_value( CUtils.dict_value_by_name( self.get_information(), self.Plugins_Info_Child_Layer_Plugins_Name, '' ) ) table.column_list.column_by_name('dsodatatype').set_value( CUtils.dict_value_by_name( self.get_information(), self.Plugins_Info_Child_Layer_Data_Type, '' ) ) table.column_list.column_by_name('dsoalphacode').set_value(CUtils.alpha_text(layer_name)) table.column_list.column_by_name('dsoaliasname').set_value(layer_alias_name) table.column_list.column_by_name('dsoparentobjid').set_value(parser.object_id) table.column_list.column_by_name('dso_ib_id').set_value(mdb_ib_id) table.column_list.column_by_name('dsometadatatext').set_value(layer_metadata_text) table.column_list.column_by_name('dsometadatajson').set_value(layer_metadata_text) table.column_list.column_by_name('dsometadataparsestatus').set_value(self.ProcStatus_InQueue) table.column_list.column_by_name('dsotagsparsestatus').set_value(self.ProcStatus_InQueue) table.column_list.column_by_name('dsodetailparsestatus').set_value(self.ProcStatus_InQueue) result = table.save_data() if not CResult.result_success(result): error_message_list.append( '图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format( layer_name, CResult.result_message(result) ) ) except Exception as error: error_message_list.append('图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(layer_name, error.__str__())) if len(error_message_list) > 0: return CResult.merge_result( self.Failure, '数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 详细情况如下: \n{1}'.format( self.file_info.file_name_with_full_path, CUtils.list_2_str(error_message_list, '', '\n', '', True) ) ) else: return CResult.merge_result( self.Success, '数据[{0}]的自定义元数据解析完毕! '.format( self.file_info.file_name_with_full_path, ) )
def process_mission(self, dataset) -> str: """ 详细算法复杂, 参见readme.md中[### 数据入库调度]章节 :param dataset: :return: """ ds_src_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_src_storage_type = dataset.value_by_name(0, 'query_storage_type', self.Storage_Type_Mix) ds_src_root_path = dataset.value_by_name(0, 'query_rootpath', '') ds_src_dir_id = dataset.value_by_name(0, 'query_ib_dir_id', '') ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '') ds_ib_directory_name = dataset.value_by_name(0, 'query_ib_relation_dir', '') ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '') ds_ib_option = dataset.value_by_name(0, 'query_ib_option', '') src_need_storage_size = self.get_storage_size(ds_ib_id, ds_src_storage_id, ds_ib_directory_name, ds_ib_option) src_path = ds_src_root_path if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): src_path = CFile.join_file(src_path, ds_ib_directory_name) src_dataset_metadata_filename = CFile.join_file( src_path, self.FileName_MetaData_Bus_21AT) CLogger().debug('入库的目录为: {0}.{1}'.format(ds_ib_id, ds_ib_directory_name)) try: # 检查所有文件与元数据是否相符 all_ib_file_or_path_existed = self.check_all_ib_file_or_path_existed( ds_ib_id) if not CResult.result_success(all_ib_file_or_path_existed): self.update_ib_result(ds_ib_id, all_ib_file_or_path_existed) return all_ib_file_or_path_existed # 将数据入库的记录保存到日志中 result = self.ib_log(ds_ib_id, ds_src_storage_id, ds_ib_directory_name) if not CResult.result_success(result): self.update_ib_result(ds_ib_id, result) return result # 如果是在核心存储或混合存储中直接入库, 则仅仅改变元数据状态即可 if CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Mix) \ or CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Core): result_ib_in_core_or_mix_storage = self.update_ib_data_status_in_core_or_mix_storage( ds_ib_id, ds_src_storage_id, ds_ib_directory_name, ds_src_dir_id) self.update_ib_result(ds_ib_id, result_ib_in_core_or_mix_storage) return result_ib_in_core_or_mix_storage # 加载目录下的待入库数据集的元数据文件 src_dataset_xml = CXml() src_dataset_type = self.Name_Default if CFile.file_or_path_exist(src_dataset_metadata_filename): src_dataset_xml.load_file(src_dataset_metadata_filename) src_dataset_type = CXml.get_element_text( src_dataset_xml.xpath_one(self.Path_MD_Bus_ProductType)) if CUtils.equal_ignore_case(src_dataset_type, ''): src_dataset_type = self.Name_Default # 获取匹配的入库模式 src_ib_schema = self.get_ib_schema(src_dataset_type, ds_ib_option) if src_ib_schema is None: result = CResult.merge_result( self.Failure, '目录为[{0}.{1}]的数据集类型为[{2}], 未找到匹配的入库模式, 请检查修正后重试!'.format( ds_ib_id, ds_ib_directory_name, src_dataset_type)) self.update_ib_result(ds_ib_id, result) return result # 计算入库的目标存储\存储根目录\目标子目录在目标存储中的副目录的标识\目标子目录\反馈消息 dest_ib_storage_id, dest_ib_root_path, desc_ib_dir_id, dest_ib_subpath, message = self.get_dest_storage( ds_ib_batch_no, src_need_storage_size, ds_ib_option, src_ib_schema, src_dataset_xml) if dest_ib_storage_id is None or dest_ib_subpath is None: result = CResult.merge_result(self.Failure, message) self.update_ib_result(ds_ib_id, result) return result dest_ib_subpath = CFile.unify(dest_ib_subpath) if CJson.json_attr_value(ds_ib_option, self.Path_IB_Switch_CheckFileLocked, self.DB_False) == self.DB_True: src_ib_files_not_locked, message = self.check_src_ib_files_not_locked( ds_src_root_path, src_path) if not src_ib_files_not_locked: result = CResult.merge_result(self.Failure, message) self.update_ib_result(ds_ib_id, result) return result proc_ib_src_path = ds_src_root_path proc_ib_dest_path = dest_ib_root_path if not CUtils.equal_ignore_case(dest_ib_subpath, ''): proc_ib_dest_path = CFile.join_file(dest_ib_root_path, dest_ib_subpath) if not CUtils.equal_ignore_case(ds_ib_directory_name, ''): proc_ib_src_path = CFile.join_file(proc_ib_src_path, ds_ib_directory_name) proc_ib_dest_path = CFile.join_file(proc_ib_dest_path, ds_ib_directory_name) # --------------------------------------------------------------至此, 数据入库前的检查处理完毕 # 移动源目录至目标目录, 如果是根目录, 则仅仅移动文件 result = self.ib_files_move( proc_ib_src_path, proc_ib_dest_path, CUtils.equal_ignore_case(ds_ib_directory_name, '')) if not CResult.result_success(result): # 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功 sub_result = self.ib_files_move( proc_ib_dest_path, proc_ib_src_path, CUtils.equal_ignore_case(ds_ib_directory_name, '')) if not CResult.result_success(sub_result): sub_result_message = CResult.result_message(sub_result) result_message = CResult.result_message(result) result = CResult.merge_result( self.Failure, '{0}\n{1}'.format(result_message, sub_result_message)) self.update_ib_result(ds_ib_id, result) return result # 将源文件的元数据, 移动至目标存储下, 如果出现异常, 则在方法内部rollback result = self.src_ib_metadata_move_to_storage( ds_ib_id, ds_src_storage_id, ds_src_dir_id, ds_ib_directory_name, dest_ib_storage_id, desc_ib_dir_id, dest_ib_subpath) if not CResult.result_success(result): # 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功 sub_result = self.ib_files_move( proc_ib_dest_path, proc_ib_src_path, CUtils.equal_ignore_case(ds_ib_directory_name, '')) if not CResult.result_success(sub_result): sub_result_message = CResult.result_message(sub_result) result_message = CResult.result_message(result) result = CResult.merge_result( self.Failure, '{0}/n{1}'.format(result_message, sub_result_message)) self.update_ib_result(ds_ib_id, result) return result result = CResult.merge_result( self.Success, '目录为[{0}.{1}]入库成功!'.format(ds_ib_id, ds_ib_directory_name)) self.update_ib_result(ds_ib_id, result) return result except Exception as error: result = CResult.merge_result( self.Failure, '目录为[{0}.{1}]入库出现异常! 错误原因为: {2}'.format( ds_ib_id, ds_ib_directory_name, error.__str__())) self.update_ib_result(ds_ib_id, result) return result
'pipe': [ { 'field': 'dsoid', 'type': 'common', 'value': '$bh' }, { 'field': 'a', 'type': 'common', 'value': '$jcqybm' }, { 'field': 'b', 'type': 'common', 'value': '$jcqymc' }, { 'field': 'dso_geo_wgs84', 'type': 'geometry', 'value': '$geometry', 'srid': 4326 } ], 'option': { 'same_field_name_match': True } } ) result = data_import.process() print(CResult.result_message(result))
def process_mission(self, dataset) -> str: """ :param dataset: :return: """ ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ds_storage_title = dataset.value_by_name(0, 'query_storage_title', '') ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '') ds_ib_directory_name = dataset.value_by_name(0, 'query_ib_relation_dir', '') ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '') ds_ib_option = CUtils.any_2_str( dataset.value_by_name(0, 'query_ib_option', '')) CLogger().debug('与第三方模块同步的目录为: {0}.{1}'.format(ds_ib_id, ds_ib_directory_name)) data_count = 0 try: module_name_list = CJson.json_attr_value( ds_ib_option, self.Path_IB_Opt_Notify_module, None) if module_name_list is None: modules_root_dir = CSys.get_metadata_data_access_modules_root_dir( ) module_file_list = CFile.file_or_subpath_of_path( modules_root_dir, '{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py)) module_name_list = list() for module_file in module_file_list: module_name_list.append(CFile.file_main_name(module_file)) sql_ib_need_notify_object = ''' select dsoid, dsoobjecttype, dsoobjectname, dso_da_result from dm2_storage_object where dso_ib_id = :ib_id ''' dataset = CFactory().give_me_db(self.get_mission_db_id()).all_row( sql_ib_need_notify_object, {'ib_id': ds_ib_id}) if dataset.is_empty(): result = CResult.merge_result( self.Success, '存储[{0}]下, 批次为[{1}]的目录[{2}]下无任何对象, 不再通知给第三方应用!'.format( ds_storage_title, ds_ib_batch_no, ds_ib_directory_name)) self.update_notify_result(ds_ib_id, result) return result CLogger().debug( '存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象等待通知给第三方应用!'.format( ds_storage_title, ds_ib_batch_no, ds_ib_directory_name, dataset.size())) data_count = dataset.size() error_message = '' for data_index in range(data_count): record_object = dataset.record(data_index) object_id = CUtils.dict_value_by_name(record_object, 'dsoid', '') object_type = CUtils.dict_value_by_name( record_object, 'dsoobjecttype', '') object_name = CUtils.dict_value_by_name( record_object, 'dsoobjectname', '') object_da_result_text = CUtils.any_2_str( CUtils.dict_value_by_name(record_object, 'dso_da_result', '')) object_da_result = CJson() object_da_result.load_json_text(object_da_result_text) for module_name in module_name_list: module_obj = CObject.create_module_instance( CSys.get_metadata_data_access_modules_root_name(), module_name, self.get_mission_db_id()) module_id = module_name module_title = CUtils.dict_value_by_name( module_obj.information(), self.Name_Title, '') module_enable = CUtils.dict_value_by_name( module_obj.information(), self.Name_Enable, True) if not module_enable: continue module_access = object_da_result.xpath_one( '{0}.{1}'.format(module_id, self.Name_Result), self.DataAccess_Forbid) module_access_memo = object_da_result.xpath_one( '{0}.{1}'.format(module_id, self.Name_Message), '') CLogger().debug( '存储[{0}]下, 批次为[{1}]的目录[{2}]下的对象[{3}], 与模块[{4}]的访问权限为[{5}]!' .format(ds_storage_title, ds_ib_batch_no, ds_ib_directory_name, object_name, module_title, module_access)) # todo(王西亚) 仔细考虑这里是否要放开, 是放开pass的, 还是放开pass和wait!!!!!! # if not \ # ( # CUtils.equal_ignore_case(module_access, self.DataAccess_Pass) # or CUtils.equal_ignore_case(module_access, self.DataAccess_Wait) # ): # continue result = module_obj.notify_object(ds_ib_id, module_access, module_access_memo, object_id, object_name, object_type, None) if not CResult.result_success(result): message = CResult.result_message(result) CLogger().debug( '存储[{0}]下, 批次为[{1}]的目录[{2}]下的对象[{3}], 与模块[{4}]的通知处理结果出现错误, 详细情况: [{5}]!' .format(ds_storage_title, ds_ib_batch_no, ds_ib_directory_name, object_name, module_title, message)) error_message = CUtils.str_append( error_message, message) if CUtils.equal_ignore_case(error_message, ''): result = CResult.merge_result( self.Success, '存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象成功通知给第三方应用!'.format( ds_storage_title, ds_ib_batch_no, ds_ib_directory_name, data_count)) self.update_notify_result(ds_ib_id, result) return result else: result = CResult.merge_result( self.Failure, '存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象在通知给第三方应用时, 部分出现错误! 错误信息如下: \n{4}' .format(ds_storage_title, ds_ib_batch_no, ds_ib_directory_name, data_count, error_message)) self.update_notify_result(ds_ib_id, result) return result except Exception as error: result = CResult.merge_result( self.Failure, '存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象通知给第三方应用时出现异常! 错误原因为: {4}!' .format(ds_storage_title, ds_ib_batch_no, ds_ib_directory_name, data_count, error.__str__())) self.update_notify_result(ds_ib_id, result) return result