def db_update_object_status(self, dso_id, process_result):
CLogger().debug(CResult.result_message(process_result))
if CResult.result_success(process_result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsotagsparsestatus = {0}
, dsolastmodifytime = now()
, dsotagsparsememo = :dsotagsparsememo
where dsoid = :dsoid
'''.format(self.ProcStatus_Finished), {
'dsoid': dso_id,
'dsotagsparsememo': CResult.result_message(process_result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsotagsparsestatus = {0}
, dsolastmodifytime = now()
, dsotagsparsememo = :dsotagsparsememo
where dsoid = :dsoid
'''.format(self.ProcStatus_Error), {
'dsoid': dso_id,
'dsotagsparsememo': CResult.result_message(process_result)
})
def update_notify_result(self, notify_id, result):
if CResult.result_success(result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsi_na_status = {0}
, dsi_na_proc_id = null
, dsi_na_proc_memo = :notify_message
, dsiproctime = now()
where dsiid = :notify_id
'''.format(self.ProcStatus_WaitConfirm), {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsi_na_status = {0}
, dsi_na_proc_id = null
, dsi_na_proc_memo = :notify_message
, dsiproctime = now()
where dsiid = :notify_id
'''.format(self.ProcStatus_Error), {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
def db_update_object_exception(self,
dso_id,
process_result,
process_status=None):
if process_status is not None:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsometadataparsestatus = :status
, dsometadataparsememo = :dsometadataparsememo
, dsolastmodifytime = now()
where dsoid = :dsoid
'''.format(self.ProcStatus_InQueue), {
'dsoid': dso_id,
'status': process_status,
'dsometadataparsememo':
CResult.result_message(process_result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsometadataparsestatus = (dsometadataparsestatus / 10 + 1) * 10 + {0}
, dsometadataparsememo = :dsometadataparsememo
, dsolastmodifytime = now()
where dsoid = :dsoid
'''.format(self.ProcStatus_InQueue), {
'dsoid': dso_id,
'dsometadataparsememo':
CResult.result_message(process_result)
})
def update_inbound_qi_result(self,
notify_id,
result,
storage_type='mix',
storage_option=None,
ib_option=None):
CLogger().debug(CResult.result_message(result))
if CResult.result_success(result):
if CUtils.equal_ignore_case(storage_type,
self.Storage_Type_InBound):
switch_inbound_after_qi_immediately_status = CUtils.equal_ignore_case(
settings.application.xpath_one(
self.path_switch(
self.Path_Setting_MetaData_QI_Switch, self.
Switch_Inbound_After_QI_Immediately_Of_IB_Storage),
self.Name_ON), self.Name_ON)
else:
switch_inbound_after_qi_immediately_status = CUtils.equal_ignore_case(
settings.application.xpath_one(
self.path_switch(
self.Path_Setting_MetaData_QI_Switch, self.
Switch_Inbound_After_QI_Immediately_Of_MIX_Storage
), self.Name_OFF), self.Name_ON)
if switch_inbound_after_qi_immediately_status:
next_status = self.IB_Status_IB_InQueue
else:
next_status = self.IB_Status_QI_Finished
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsiStatus = {0}, dsiprocmemo = :notify_message, dsiproctime = now()
where dsiid = :notify_id
'''.format(next_status), {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsiprocmemo = :notify_message, dsiproctime = now()
where dsiid = :notify_id
''', {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
def sync(self) -> str:
try:
result = self.process_metadata_bus_dict()
if not CResult.result_success(result):
return CResult.merge_result(
self.Failure, '卫星数据的业务元数据的详细内容解析出错!原因为{0}'.format(
CResult.result_message(result)))
main_result = self.process_main_table()
metadata_result = self.process_metadata_table()
ndi_result = self.process_ndi_table()
if not CResult.result_success(main_result):
return main_result
elif not CResult.result_success(metadata_result):
return metadata_result
elif not CResult.result_success(ndi_result):
return ndi_result
else:
return CResult.merge_result(
self.Success, '对象[{0}]的同步成功! '.format(self._obj_name))
except Exception as error:
return CResult.merge_result(
self.Failure,
'数据检索分发模块在进行数据同步时出现错误:同步的对象[{0}]在处理时出现异常, 详细情况: [{1}]!'.format(
self._obj_name, error.__str__()))
def update_dir_status(self, dir_id, result, status=None):
if status is not None:
sql_update_directory_status = '''
update dm2_storage_directory
set
dsdscanstatus = :status,
dsdscanmemo = :memo,
dsdlastmodifytime = now()
where dsdid = :dsdid
'''
elif CResult.result_success(result):
sql_update_directory_status = '''
update dm2_storage_directory
set
dsdscanstatus = {0},
dsdscanmemo = :memo,
dsdlastmodifytime = now()
where dsdid = :dsdid
'''.format(self.ProcStatus_Finished)
else:
sql_update_directory_status = '''
update dm2_storage_directory
set
dsdscanstatus = (dsdscanstatus / 10 + 1) * 10 + {0},
dsdscanmemo = :memo,
dsdlastmodifytime = now()
where dsdid = :dsdid
'''.format(self.ProcStatus_InQueue)
params = dict()
params['dsdid'] = dir_id
params['memo'] = CResult.result_message(result)
params['status'] = status
CFactory().give_me_db(self.get_mission_db_id()).execute(sql_update_directory_status, params)
def process(self) -> str:
"""
todo 负责人 赵宇飞 在这里提取影像数据的空间信息, 以文件形式存储在self.file_content.work_root_dir下
注意返回的串中有空间信息的文件名
注意: 如果出现内存泄漏现象, 则使用新建进程提取元数据, 放置到文件中, 在本进程中解析元数据!!!
:return:
"""
result_process = self.process_raster()
if CResult.result_success(result_process):
file_path = self.file_content.work_root_dir
dict_temp_file_name = {
self.Name_Native_Center: '{0}_native_center.wkt'.format(self.object_name),
self.Name_Native_BBox: '{0}_native_bbox.wkt'.format(self.object_name),
self.Name_Native_Geom: '{0}_native_geom.wkt'.format(self.object_name),
self.Name_Wgs84_Center: '{0}_wgs84_center.wkt'.format(self.object_name),
self.Name_Wgs84_BBox: '{0}_wgs84_bbox.wkt'.format(self.object_name),
self.Name_Wgs84_Geom: '{0}_wgs84_geom.wkt'.format(self.object_name)
}
dict_temp_prj_name = {
self.Name_Prj_Wkt: CResult.result_info(result_process, self.Name_Prj_Wkt, None),
self.Name_Prj_Proj4: CResult.result_info(result_process, self.Name_Prj_Proj4, None),
self.Name_Prj_Project: CResult.result_info(result_process, self.Name_Prj_Project, None),
self.Name_Prj_Coordinate: CResult.result_info(result_process, self.Name_Prj_Coordinate, None),
self.Name_Prj_Source: CResult.result_info(result_process, self.Name_Prj_Source, None),
self.Name_Prj_Zone: CResult.result_info(result_process, self.Name_Prj_Zone, None),
self.Name_Prj_Degree: CResult.result_info(result_process, self.Name_Prj_Degree, None)
}
result = CResult.merge_result(self.Success, '处理完毕!')
for file_type, file_name in dict_temp_file_name.items():
result = CResult.merge_result_info(result, file_type, CFile.join_file(file_path, file_name))
for prj_type, prj_name in dict_temp_prj_name.items():
result = CResult.merge_result_info(result, prj_type, prj_name)
else:
result = CResult.merge_result(self.Failure, CResult.result_message(result_process))
return result
def __update_layer_update_result(self, deploy_id, result):
if CResult.result_success(result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dp_v_qfg_layer
set dpStatus = 0, dpprocessresult = :message
where dpid = :id
''', {'id': deploy_id, 'message': CResult.result_message(result)}
)
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dp_v_qfg_layer
set dpStatus = 21, dpprocessresult = :message
where dpid = :id
''', {'id': deploy_id, 'message': CResult.result_message(result)}
)
def update_inbound_na_progress(self, notify_id, message):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsi_na_proc_memo = :notify_message, dsiproctime = now()
where dsiid = :notify_id
''', {
'notify_id': notify_id,
'notify_message': CResult.result_message(message)
})
def update_sync_result(self, na_id, result):
if CResult.result_success(result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_obj_na
set dson_notify_status = {0}, dson_notify_proc_id = null, dson_notify_proc_memo = :proc_message
where dsonid = :id
'''.format(self.ProcStatus_Finished), {
'id': na_id,
'proc_message': CResult.result_message(result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_obj_na
set dson_notify_status = {0}, dson_notify_proc_id = null, dson_notify_proc_memo = :proc_message
where dsonid = :id
'''.format(self.ProcStatus_Error), {
'id': na_id,
'proc_message': CResult.result_message(result)
})
def update_inbound_na_result(self, notify_id, result):
CLogger().debug(CResult.result_message(result))
if CResult.result_success(result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsi_na_status = {0}, dsi_na_proc_memo = :notify_message, dsiproctime = now()
where dsiid = :notify_id
'''.format(self.ProcStatus_Finished), {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsi_na_proc_memo = :notify_message, dsiproctime = now()
where dsiid = :notify_id
''', {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
def update_inbound_qi_result(self, notify_id, result):
CLogger().debug(CResult.result_message(result))
if CResult.result_success(result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsiStatus = {0}, dsiprocmemo = :notify_message
where dsiid = :notify_id
'''.format(self.IB_Status_QI_Processing), {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsiStatus = {0}, dsiprocmemo = :notify_message
where dsiid = :notify_id
'''.format(self.IB_Status_QI_Error), {
'notify_id': notify_id,
'notify_message': CResult.result_message(result)
})
def update_ib_result(self, ib_id, result):
if CResult.result_success(result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsistatus = {0}, dsiprocmemo = :ib_message, dsi_na_status = {1}
where dsiid = :ib_id
'''.format(self.ProcStatus_Finished, self.ProcStatus_InQueue),
{
'ib_id': ib_id,
'ib_message': CResult.result_message(result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_inbound
set dsistatus = {0}, dsiprocmemo = :ib_message
where dsiid = :ib_id
'''.format(self.IB_Status_IB_Error), {
'ib_id': ib_id,
'ib_message': CResult.result_message(result)
})
def db_update_object_status(self, dso_id, result, status=None):
if status is not None:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsodetailparsestatus = :status
, dsolastmodifytime = now()
, dsodetailparsememo = :dsodetailparsememo
where dsoid = :dsoid
''', {
'dsoid': dso_id,
'dsodetailparsememo': CResult.result_message(result),
'status': status
})
elif CResult.result_success(result):
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsodetailparsestatus = {0}
, dsolastmodifytime = now()
, dsodetailparsememo = :dsodetailparsememo
where dsoid = :dsoid
'''.format(self.ProcStatus_Finished), {
'dsoid': dso_id,
'dsodetailparsememo': CResult.result_message(result)
})
else:
CFactory().give_me_db(self.get_mission_db_id()).execute(
'''
update dm2_storage_object
set dsodetailparsestatus = (dsodetailparsestatus / 10 + 1) * 10 + {0},
, dsolastmodifytime = now()
, dsodetailparsememo = :dsodetailparsememo
where dsoid = :dsoid
'''.format(self.ProcStatus_InQueue), {
'dsoid': dso_id,
'dsodetailparsememo': CResult.result_message(result)
})
def process_mission(self, dataset):
dso_id = dataset.value_by_name(0, 'dsoid', '')
dso_data_type = dataset.value_by_name(0, 'dsodatatype', '')
dso_object_type = dataset.value_by_name(0, 'dsoobjecttype', '')
dso_object_name = dataset.value_by_name(0, 'dsoobjectname', '')
dso_object_da_content = CUtils.any_2_str(dataset.value_by_name(0, 'dso_da_result', ''))
dso_object_quality = dataset.value_by_name(0, 'dso_quality', '')
ds_retry_times = dataset.value_by_name(0, 'retry_times', 0)
if ds_retry_times >= self.abnormal_job_retry_times():
ds_last_process_memo = CUtils.any_2_str(dataset.value_by_name(0, 'last_process_memo', None))
process_result = CResult.merge_result(
self.Failure,
'{0}, \n系统已经重试{1}次, 仍然未能解决, 请人工检查修正后重试!'.format(
ds_last_process_memo,
ds_retry_times
)
)
self.update_status(dso_id, None, process_result, self.ProcStatus_Error)
return process_result
dso_quality = CXml()
dso_quality.load_xml(dso_object_quality)
dso_da_json = CJson()
dso_da_json.load_json_text(dso_object_da_content)
CLogger().debug(
'开始处理对象: {0}.{1}.{2}.{3}对各个子系统的支撑能力'.format(dso_id, dso_data_type, dso_object_type, dso_object_name))
try:
modules_root_dir = CSys.get_metadata_data_access_modules_root_dir()
modules_file_list = CFile.file_or_subpath_of_path(
modules_root_dir,
'{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py)
)
for file_name_without_path in modules_file_list:
file_main_name = CFile.file_main_name(file_name_without_path)
# 判断模块的可访问是否已经被人工审批, 如果人工审批, 则这里不再计算和覆盖
module_access = dso_da_json.xpath_one(
'{0}.{1}'.format(file_main_name, self.Name_Audit),
self.Name_System
)
if CUtils.equal_ignore_case(module_access, self.Name_User):
continue
try:
module_obj = CObject.create_module_instance(
CSys.get_metadata_data_access_modules_root_name(),
file_main_name,
self.get_mission_db_id()
)
module_title = CUtils.dict_value_by_name(module_obj.information(), self.Name_Title, '')
result = CUtils.any_2_str(module_obj.access(dso_id,
dso_object_name,
dso_data_type,
dso_quality)
)
if CResult.result_success(result):
module_access = CResult.result_info(result, self.Name_Access, self.DataAccess_Forbid)
else:
CLogger().debug('模块[{0}]解析出现错误, 系统将忽略本模块, 继续处理下一个!'.format(file_main_name))
module_access = self.DataAccess_Unknown
module_access_message = CResult.result_message(result)
module_obj = {self.Name_Audit: self.Name_System, self.Name_Result: module_access,
self.Name_Title: module_title, self.Name_Message: module_access_message}
dso_da_json.set_value_of_name(file_main_name, module_obj)
except Exception as error:
CLogger().debug('模块[{0}]解析出现异常, 原因为[{1}], 请检查!'.format(file_main_name, error.__str__()))
module_access = self.DataAccess_Unknown
module_access_message = '模块[{0}]解析出现异常, 原因为[{1}], 请检查!'.format(file_main_name, error.__str__())
module_obj = {self.Name_Audit: self.Name_System, self.Name_Result: module_access,
self.Name_Title: file_main_name, self.Name_Message: module_access_message}
dso_da_json.set_value_of_name(file_main_name, module_obj)
process_result = CResult.merge_result(
self.Success,
'对象[{0}.{1}]访问权限解析成功!'.format(dso_id, dso_object_name)
)
self.update_status(
dso_id,
dso_da_json.to_json(),
process_result
)
return process_result
except Exception as error:
process_result = CResult.merge_result(
self.Failure,
'对象[{0}.{1}]访问权限解析出错, 原因为[{2}]!'.format(dso_id, dso_object_name, error.__str__())
)
self.update_status(dso_id, None, process_result)
return process_result
def parser_metadata_custom(self, parser: CMetaDataParser) -> str:
"""
自定义的元数据解析, 在所有质检和其他处理之后触发
:param parser:
:return:
"""
meta_data_json = parser.metadata.metadata_json()
if meta_data_json is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 无法获取JSON格式的元数据! '.format(
self.file_info.file_name_with_full_path,
)
)
json_data_source = meta_data_json.xpath_one('datasource', None)
layer_list = meta_data_json.xpath_one(self.Name_Layers, None)
if layer_list is None:
return CResult.merge_result(
self.Success,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 元数据中无法找到layers节点! '.format(
self.file_info.file_name_with_full_path,
)
)
mdb_ib_id = CFactory().give_me_db(self.file_info.db_server_id).one_value(
'''
select dso_ib_id
from dm2_storage_object
where dsoid = :object_id
''',
{
'object_id': parser.object_id
}
)
error_message_list = []
table = CTable()
table.load_info(self.file_info.db_server_id, self.TableName_DM_Object)
for layer in layer_list:
layer_name = CUtils.dict_value_by_name(layer, self.Name_Name, '')
if CUtils.equal_ignore_case(layer_name, ''):
continue
layer_alias_name = CUtils.dict_value_by_name(layer, self.Name_Description, layer_name)
layer_metadata_json = CJson()
layer_metadata_json.set_value_of_name('datasource', json_data_source)
layer_metadata_json.set_value_of_name('layer_count', 1)
layer_metadata_json.set_value_of_name('layers', [layer])
layer_metadata_text = layer_metadata_json.to_json()
try:
sql_find_layer_existed = '''
select dsoid as layer_id_existed
from dm2_storage_object
where upper(dsoobjectname) = upper(:layer_name)
and dsoparentobjid = :object_id
'''
layer_id_existed = CFactory().give_me_db(self.file_info.db_server_id).one_value(
sql_find_layer_existed,
{
'layer_name': layer_name,
'object_id': parser.object_id
}
)
if layer_id_existed is None:
layer_id_existed = CUtils.one_id()
table.column_list.reset()
table.column_list.column_by_name('dsoid').set_value(layer_id_existed)
table.column_list.column_by_name('dsoobjectname').set_value(layer_name)
table.column_list.column_by_name('dsoobjecttype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Plugins_Name,
''
)
)
table.column_list.column_by_name('dsodatatype').set_value(
CUtils.dict_value_by_name(
self.get_information(),
self.Plugins_Info_Child_Layer_Data_Type,
''
)
)
table.column_list.column_by_name('dsoalphacode').set_value(CUtils.alpha_text(layer_name))
table.column_list.column_by_name('dsoaliasname').set_value(layer_alias_name)
table.column_list.column_by_name('dsoparentobjid').set_value(parser.object_id)
table.column_list.column_by_name('dso_ib_id').set_value(mdb_ib_id)
table.column_list.column_by_name('dsometadatatext').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadatajson').set_value(layer_metadata_text)
table.column_list.column_by_name('dsometadataparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsotagsparsestatus').set_value(self.ProcStatus_InQueue)
table.column_list.column_by_name('dsodetailparsestatus').set_value(self.ProcStatus_InQueue)
result = table.save_data()
if not CResult.result_success(result):
error_message_list.append(
'图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(
layer_name,
CResult.result_message(result)
)
)
except Exception as error:
error_message_list.append('图层[{0}]的创建过程出现错误, 详细信息为: {1}'.format(layer_name, error.__str__()))
if len(error_message_list) > 0:
return CResult.merge_result(
self.Failure,
'数据[{0}]的质检和空间等元数据解析完毕, 但子图层解析有误, 详细情况如下: \n{1}'.format(
self.file_info.file_name_with_full_path,
CUtils.list_2_str(error_message_list, '', '\n', '', True)
)
)
else:
return CResult.merge_result(
self.Success,
'数据[{0}]的自定义元数据解析完毕! '.format(
self.file_info.file_name_with_full_path,
)
)
def process_mission(self, dataset) -> str:
"""
详细算法复杂, 参见readme.md中[### 数据入库调度]章节
:param dataset:
:return:
"""
ds_src_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_src_storage_type = dataset.value_by_name(0, 'query_storage_type',
self.Storage_Type_Mix)
ds_src_root_path = dataset.value_by_name(0, 'query_rootpath', '')
ds_src_dir_id = dataset.value_by_name(0, 'query_ib_dir_id', '')
ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '')
ds_ib_directory_name = dataset.value_by_name(0,
'query_ib_relation_dir',
'')
ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '')
ds_ib_option = dataset.value_by_name(0, 'query_ib_option', '')
src_need_storage_size = self.get_storage_size(ds_ib_id,
ds_src_storage_id,
ds_ib_directory_name,
ds_ib_option)
src_path = ds_src_root_path
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
src_path = CFile.join_file(src_path, ds_ib_directory_name)
src_dataset_metadata_filename = CFile.join_file(
src_path, self.FileName_MetaData_Bus_21AT)
CLogger().debug('入库的目录为: {0}.{1}'.format(ds_ib_id,
ds_ib_directory_name))
try:
# 检查所有文件与元数据是否相符
all_ib_file_or_path_existed = self.check_all_ib_file_or_path_existed(
ds_ib_id)
if not CResult.result_success(all_ib_file_or_path_existed):
self.update_ib_result(ds_ib_id, all_ib_file_or_path_existed)
return all_ib_file_or_path_existed
# 将数据入库的记录保存到日志中
result = self.ib_log(ds_ib_id, ds_src_storage_id,
ds_ib_directory_name)
if not CResult.result_success(result):
self.update_ib_result(ds_ib_id, result)
return result
# 如果是在核心存储或混合存储中直接入库, 则仅仅改变元数据状态即可
if CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Mix) \
or CUtils.equal_ignore_case(ds_src_storage_type, self.Storage_Type_Core):
result_ib_in_core_or_mix_storage = self.update_ib_data_status_in_core_or_mix_storage(
ds_ib_id, ds_src_storage_id, ds_ib_directory_name,
ds_src_dir_id)
self.update_ib_result(ds_ib_id,
result_ib_in_core_or_mix_storage)
return result_ib_in_core_or_mix_storage
# 加载目录下的待入库数据集的元数据文件
src_dataset_xml = CXml()
src_dataset_type = self.Name_Default
if CFile.file_or_path_exist(src_dataset_metadata_filename):
src_dataset_xml.load_file(src_dataset_metadata_filename)
src_dataset_type = CXml.get_element_text(
src_dataset_xml.xpath_one(self.Path_MD_Bus_ProductType))
if CUtils.equal_ignore_case(src_dataset_type, ''):
src_dataset_type = self.Name_Default
# 获取匹配的入库模式
src_ib_schema = self.get_ib_schema(src_dataset_type, ds_ib_option)
if src_ib_schema is None:
result = CResult.merge_result(
self.Failure,
'目录为[{0}.{1}]的数据集类型为[{2}], 未找到匹配的入库模式, 请检查修正后重试!'.format(
ds_ib_id, ds_ib_directory_name, src_dataset_type))
self.update_ib_result(ds_ib_id, result)
return result
# 计算入库的目标存储\存储根目录\目标子目录在目标存储中的副目录的标识\目标子目录\反馈消息
dest_ib_storage_id, dest_ib_root_path, desc_ib_dir_id, dest_ib_subpath, message = self.get_dest_storage(
ds_ib_batch_no, src_need_storage_size, ds_ib_option,
src_ib_schema, src_dataset_xml)
if dest_ib_storage_id is None or dest_ib_subpath is None:
result = CResult.merge_result(self.Failure, message)
self.update_ib_result(ds_ib_id, result)
return result
dest_ib_subpath = CFile.unify(dest_ib_subpath)
if CJson.json_attr_value(ds_ib_option,
self.Path_IB_Switch_CheckFileLocked,
self.DB_False) == self.DB_True:
src_ib_files_not_locked, message = self.check_src_ib_files_not_locked(
ds_src_root_path, src_path)
if not src_ib_files_not_locked:
result = CResult.merge_result(self.Failure, message)
self.update_ib_result(ds_ib_id, result)
return result
proc_ib_src_path = ds_src_root_path
proc_ib_dest_path = dest_ib_root_path
if not CUtils.equal_ignore_case(dest_ib_subpath, ''):
proc_ib_dest_path = CFile.join_file(dest_ib_root_path,
dest_ib_subpath)
if not CUtils.equal_ignore_case(ds_ib_directory_name, ''):
proc_ib_src_path = CFile.join_file(proc_ib_src_path,
ds_ib_directory_name)
proc_ib_dest_path = CFile.join_file(proc_ib_dest_path,
ds_ib_directory_name)
# --------------------------------------------------------------至此, 数据入库前的检查处理完毕
# 移动源目录至目标目录, 如果是根目录, 则仅仅移动文件
result = self.ib_files_move(
proc_ib_src_path, proc_ib_dest_path,
CUtils.equal_ignore_case(ds_ib_directory_name, ''))
if not CResult.result_success(result):
# 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功
sub_result = self.ib_files_move(
proc_ib_dest_path, proc_ib_src_path,
CUtils.equal_ignore_case(ds_ib_directory_name, ''))
if not CResult.result_success(sub_result):
sub_result_message = CResult.result_message(sub_result)
result_message = CResult.result_message(result)
result = CResult.merge_result(
self.Failure,
'{0}\n{1}'.format(result_message, sub_result_message))
self.update_ib_result(ds_ib_id, result)
return result
# 将源文件的元数据, 移动至目标存储下, 如果出现异常, 则在方法内部rollback
result = self.src_ib_metadata_move_to_storage(
ds_ib_id, ds_src_storage_id, ds_src_dir_id,
ds_ib_directory_name, dest_ib_storage_id, desc_ib_dir_id,
dest_ib_subpath)
if not CResult.result_success(result):
# 利用相同的方法, 把移动的数据, 重新移动回原目录, 这里理论上应该100%成功
sub_result = self.ib_files_move(
proc_ib_dest_path, proc_ib_src_path,
CUtils.equal_ignore_case(ds_ib_directory_name, ''))
if not CResult.result_success(sub_result):
sub_result_message = CResult.result_message(sub_result)
result_message = CResult.result_message(result)
result = CResult.merge_result(
self.Failure,
'{0}/n{1}'.format(result_message, sub_result_message))
self.update_ib_result(ds_ib_id, result)
return result
result = CResult.merge_result(
self.Success,
'目录为[{0}.{1}]入库成功!'.format(ds_ib_id, ds_ib_directory_name))
self.update_ib_result(ds_ib_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure, '目录为[{0}.{1}]入库出现异常! 错误原因为: {2}'.format(
ds_ib_id, ds_ib_directory_name, error.__str__()))
self.update_ib_result(ds_ib_id, result)
return result
'pipe': [
{
'field': 'dsoid',
'type': 'common',
'value': '$bh'
},
{
'field': 'a',
'type': 'common',
'value': '$jcqybm'
},
{
'field': 'b',
'type': 'common',
'value': '$jcqymc'
},
{
'field': 'dso_geo_wgs84',
'type': 'geometry',
'value': '$geometry',
'srid': 4326
}
],
'option': {
'same_field_name_match': True
}
}
)
result = data_import.process()
print(CResult.result_message(result))
def process_mission(self, dataset) -> str:
"""
:param dataset:
:return:
"""
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ds_storage_title = dataset.value_by_name(0, 'query_storage_title', '')
ds_ib_id = dataset.value_by_name(0, 'query_ib_id', '')
ds_ib_directory_name = dataset.value_by_name(0,
'query_ib_relation_dir',
'')
ds_ib_batch_no = dataset.value_by_name(0, 'query_ib_batchno', '')
ds_ib_option = CUtils.any_2_str(
dataset.value_by_name(0, 'query_ib_option', ''))
CLogger().debug('与第三方模块同步的目录为: {0}.{1}'.format(ds_ib_id,
ds_ib_directory_name))
data_count = 0
try:
module_name_list = CJson.json_attr_value(
ds_ib_option, self.Path_IB_Opt_Notify_module, None)
if module_name_list is None:
modules_root_dir = CSys.get_metadata_data_access_modules_root_dir(
)
module_file_list = CFile.file_or_subpath_of_path(
modules_root_dir,
'{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py))
module_name_list = list()
for module_file in module_file_list:
module_name_list.append(CFile.file_main_name(module_file))
sql_ib_need_notify_object = '''
select dsoid, dsoobjecttype, dsoobjectname, dso_da_result
from dm2_storage_object
where dso_ib_id = :ib_id
'''
dataset = CFactory().give_me_db(self.get_mission_db_id()).all_row(
sql_ib_need_notify_object, {'ib_id': ds_ib_id})
if dataset.is_empty():
result = CResult.merge_result(
self.Success,
'存储[{0}]下, 批次为[{1}]的目录[{2}]下无任何对象, 不再通知给第三方应用!'.format(
ds_storage_title, ds_ib_batch_no,
ds_ib_directory_name))
self.update_notify_result(ds_ib_id, result)
return result
CLogger().debug(
'存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象等待通知给第三方应用!'.format(
ds_storage_title, ds_ib_batch_no, ds_ib_directory_name,
dataset.size()))
data_count = dataset.size()
error_message = ''
for data_index in range(data_count):
record_object = dataset.record(data_index)
object_id = CUtils.dict_value_by_name(record_object, 'dsoid',
'')
object_type = CUtils.dict_value_by_name(
record_object, 'dsoobjecttype', '')
object_name = CUtils.dict_value_by_name(
record_object, 'dsoobjectname', '')
object_da_result_text = CUtils.any_2_str(
CUtils.dict_value_by_name(record_object, 'dso_da_result',
''))
object_da_result = CJson()
object_da_result.load_json_text(object_da_result_text)
for module_name in module_name_list:
module_obj = CObject.create_module_instance(
CSys.get_metadata_data_access_modules_root_name(),
module_name, self.get_mission_db_id())
module_id = module_name
module_title = CUtils.dict_value_by_name(
module_obj.information(), self.Name_Title, '')
module_enable = CUtils.dict_value_by_name(
module_obj.information(), self.Name_Enable, True)
if not module_enable:
continue
module_access = object_da_result.xpath_one(
'{0}.{1}'.format(module_id, self.Name_Result),
self.DataAccess_Forbid)
module_access_memo = object_da_result.xpath_one(
'{0}.{1}'.format(module_id, self.Name_Message), '')
CLogger().debug(
'存储[{0}]下, 批次为[{1}]的目录[{2}]下的对象[{3}], 与模块[{4}]的访问权限为[{5}]!'
.format(ds_storage_title, ds_ib_batch_no,
ds_ib_directory_name, object_name,
module_title, module_access))
# todo(王西亚) 仔细考虑这里是否要放开, 是放开pass的, 还是放开pass和wait!!!!!!
# if not \
# (
# CUtils.equal_ignore_case(module_access, self.DataAccess_Pass)
# or CUtils.equal_ignore_case(module_access, self.DataAccess_Wait)
# ):
# continue
result = module_obj.notify_object(ds_ib_id, module_access,
module_access_memo,
object_id, object_name,
object_type, None)
if not CResult.result_success(result):
message = CResult.result_message(result)
CLogger().debug(
'存储[{0}]下, 批次为[{1}]的目录[{2}]下的对象[{3}], 与模块[{4}]的通知处理结果出现错误, 详细情况: [{5}]!'
.format(ds_storage_title, ds_ib_batch_no,
ds_ib_directory_name, object_name,
module_title, message))
error_message = CUtils.str_append(
error_message, message)
if CUtils.equal_ignore_case(error_message, ''):
result = CResult.merge_result(
self.Success,
'存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象成功通知给第三方应用!'.format(
ds_storage_title, ds_ib_batch_no, ds_ib_directory_name,
data_count))
self.update_notify_result(ds_ib_id, result)
return result
else:
result = CResult.merge_result(
self.Failure,
'存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象在通知给第三方应用时, 部分出现错误! 错误信息如下: \n{4}'
.format(ds_storage_title, ds_ib_batch_no,
ds_ib_directory_name, data_count, error_message))
self.update_notify_result(ds_ib_id, result)
return result
except Exception as error:
result = CResult.merge_result(
self.Failure,
'存储[{0}]下, 批次为[{1}]的目录[{2}]下有[{3}]个对象通知给第三方应用时出现异常! 错误原因为: {4}!'
.format(ds_storage_title, ds_ib_batch_no, ds_ib_directory_name,
data_count, error.__str__()))
self.update_notify_result(ds_ib_id, result)
return result