def __prepare_where_condition(self, column: CColumn, sql_text: str,
sql_params: dict):
column_type = self.__database.db_column_type_by_name(
column.db_column_type)
if CUtils.equal_ignore_case(column_type.set_value_method,
self.DB_Column_Set_Method_Param):
sql_text = CUtils.str_append(
sql_text, '{0}=:{1}'.format(column.name,
self.__param_name(column.name)),
' and ')
sql_params[self.__param_name(column.name)] = CUtils.any_2_str(
CUtils.dict_value_by_name(column.value, self.Name_Text, ''))
elif CUtils.equal_ignore_case(column_type.set_value_method,
self.DB_Column_Set_Method_Function):
sql_text = CUtils.str_append(
sql_text, '{0}=:{1}'.format(column.name,
self.__param_name(column.name)),
' and ')
sql_params[self.__param_name(
column.name)] = CUtils.replace_placeholder(
column_type.set_value_template,
dict({
self.Name_Value,
CUtils.any_2_str(
CUtils.dict_value_by_name(column.value,
self.Name_Text, ''))
}))
return sql_text, sql_params
def __import_each_record(self, data_source: CDataSetSeqReader, data_target: CTable) -> str:
data_target.column_list.reset()
data_source_record = data_source.record_as_dict()
for column_index in range(data_target.column_list.size()):
column_obj = data_target.column_list.column_by_index(column_index)
column_name = column_obj.name
column_data_set_method = self.__find_column_data_set_method(column_obj, data_source)
if column_data_set_method is None:
continue
column_value_set_type = CUtils.dict_value_by_name(column_data_set_method, self.Name_Type, self.Name_Common)
if CUtils.equal_ignore_case(column_value_set_type, self.Name_SQL):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
column_obj.set_sql(column_value_template)
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Geometry):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
column_obj.set_geometry(
column_value_template,
CUtils.dict_value_by_name(column_data_set_method, self.Name_Srid, None)
)
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Array):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, None)
if isinstance(column_value_template, list):
column_obj.set_array(column_value_template)
else:
column_value_template = CUtils.any_2_str(column_value_template)
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template,
data_source_record)
column_obj.set_array_str(column_value_template)
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_Null):
column_obj.set_null()
elif CUtils.equal_ignore_case(column_value_set_type, self.Name_File):
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
file_format = CUtils.dict_value_by_name(column_data_set_method, self.Name_Format, self.Name_Binary)
file_format = self.replace_placeholder(file_format)
file_format = CUtils.replace_placeholder(file_format, data_source.record_as_dict())
file_encoding = CUtils.dict_value_by_name(column_data_set_method, self.Name_Encoding,
self.Encoding_UTF8)
file_encoding = self.replace_placeholder(file_encoding)
file_encoding = CUtils.replace_placeholder(file_encoding, data_source_record)
column_obj.set_value_from_file(column_value_template, file_format, file_encoding)
else:
column_value_template = CUtils.dict_value_by_name(column_data_set_method, self.Name_Value, '')
column_value_template = self.replace_placeholder(column_value_template)
column_value_template = CUtils.replace_placeholder(column_value_template, data_source_record)
column_obj.set_value(column_value_template)
data_target.save_data()
return CResult.merge_result(self.Success, '数据入库成功!')
def execute(self) -> str:
inbound_ib_n_list = CFactory().give_me_db(
self.get_mission_db_id()).all_row('''
select
dsiid as query_ib_id
, dsiotheroption as query_ib_option
, dsidirectory as query_ib_relation_dir
, dsidirectoryid as query_ib_dir_id
, dsitargetstorageid as query_ib_target_storage_id
from dm2_storage_inbound
where dsi_na_status = {0}
'''.format(self.ProcStatus_WaitConfirm))
if inbound_ib_n_list.is_empty():
return CResult.merge_result(CResult.Success, '本次没有需要检查的通知任务！')
for data_index in range(inbound_ib_n_list.size()):
ds_ib_id = inbound_ib_n_list.value_by_name(data_index,
'query_ib_id', '')
ds_ib_option = CUtils.any_2_str(
inbound_ib_n_list.value_by_name(data_index, 'query_ib_option',
''))
ds_ib_directory_name = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_relation_dir', '')
ds_ib_directory_id = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_dir_id', '')
ds_ib_target_storage_id = inbound_ib_n_list.value_by_name(
data_index, 'query_ib_target_storage_id', '')
module_name_list = CJson.json_attr_value(
ds_ib_option, self.Path_IB_Opt_Notify_module, None)
if module_name_list is None:
modules_root_dir = CSys.get_metadata_data_access_modules_root_dir(
)
module_file_list = CFile.file_or_subpath_of_path(
modules_root_dir,
'{0}_*.{1}'.format(self.Name_Module, self.FileExt_Py))
module_name_list = list()
for module_file in module_file_list:
module_name_list.append(CFile.file_main_name(module_file))
CLogger().debug('正在检查入库批次[ds_ib_id]的通知进度...'.format(ds_ib_id))
try:
# 所有通知对象的统计数
sql_record_total_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
''', {
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_total_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_total_count, {'ib_id': ds_ib_id}, 0)
if record_total_count == 0:
self.update_inbound_na_result(
ds_ib_id,
CResult.merge_result(
self.Failure,
'入库任务下没有可通知的数据, 请检查异常情况! '.format(ds_ib_id)))
continue
# 已经完成的通知对象的统计数, 包括正常完成和错误的
sql_record_finished_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_notify_status in ({0}, {1})
and dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
'''.format(self.ProcStatus_Finished,
self.ProcStatus_Error),
{
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_finished_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_finished_count, {'ib_id': ds_ib_id}, 0)
# 错误的记录数
sql_record_error_count = CUtils.replace_placeholder(
'''
select count(*)
from dm2_storage_obj_na
where dson_notify_status = {0}
and dson_app_id in ($module_name_list)
and dson_object_id in (
select dsoid
from dm2_storage_object
where dso_ib_id = :ib_id
)
'''.format(self.ProcStatus_Error), {
'module_name_list':
CUtils.list_2_str(module_name_list, "'", ',', "'")
})
record_error_count = CFactory().give_me_db(
self.get_mission_db_id()).one_value(
sql_record_error_count, {'ib_id': ds_ib_id}, 0)
if record_total_count != record_finished_count:
message = '入库任务[{0}]下的数据正在通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个...'.format(
ds_ib_id, record_total_count, record_finished_count,
record_error_count)
CLogger().debug(message)
self.update_inbound_na_progress(
ds_ib_id, CResult.merge_result(self.Failure, message))
else:
message = '入库任务[{0}]下的数据已经通知其他子系统, 共有[{1}]个, 已处理[{2}]个, 失败[{3}]个, 请检查修正! '.format(
ds_ib_id, record_total_count, record_finished_count,
record_error_count)
CLogger().debug(message)
self.update_inbound_na_result(
ds_ib_id, CResult.merge_result(self.Success, message))
except Exception as error:
self.update_inbound_na_result(
ds_ib_id,
CResult.merge_result(
self.Failure,
'入库任务下的数据通知其他子系统过程中出现异常情况, 详细错误信息为: [{1}]'.format(
ds_ib_id, error.__str__())))
continue
return CResult.merge_result(self.Success, '本次通知监控任务成功结束！')
def process_vector(self) -> str:
try:
# file_name_with_full_path = r'D:\test\vector_test\石嘴山市-3xq.json'
# file_main_name = CFile.file_main_name(file_name_with_full_path)
# file_path = CFile.file_path(file_name_with_full_path)
xml_obj = self.metadata.metadata_xml()
# <editor-fold desc="1.空间坐标信息">
wkt_info = 'POLYGON((${min_x} ${max_y},${max_x} ${max_y},' \
'${max_x} ${min_y},${min_x} ${min_y},${min_x} ${max_y}))'
# 四至坐标
native_max_x = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MaxLon'))
native_max_y = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MaxLat'))
native_min_x = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MinLon'))
native_min_y = CUtils.to_decimal(
xml_obj.get_element_text_by_xpath_one('/TileMetadata/MinLat'))
if (native_max_x is None) or (native_max_y is None) \
or (native_min_x is None) or (native_min_y is None):
native_center_wkt = None
native_bbox_wkt = None
geom_native_wkt = None
else:
dict_native = {
'max_x': CUtils.any_2_str(native_max_x),
'max_y': CUtils.any_2_str(native_max_y),
'min_x': CUtils.any_2_str(native_min_x),
'min_y': CUtils.any_2_str(native_min_y)
}
# 中心坐标
center_x = (native_max_x - native_min_x) / 2 + native_min_x
center_y = (native_max_y - native_min_y) / 2 + native_min_y
native_center_wkt = 'POINT({0} {1})'.format(center_x, center_y)
# 外边框、外包框
native_bbox_wkt = CUtils.replace_placeholder(
wkt_info[:], dict_native)
geom_native_wkt = native_bbox_wkt
file_path = self.file_content.work_root_dir
file_main_name = self.object_name
native_center_filepath = CFile.join_file(
file_path, file_main_name + '_native_center.wkt')
CFile.str_2_file(native_center_wkt, native_center_filepath)
native_bbox_filepath = CFile.join_file(
file_path, file_main_name + '_native_bbox.wkt')
CFile.str_2_file(native_bbox_wkt, native_bbox_filepath)
native_geom_filepath = CFile.join_file(
file_path, file_main_name + '_native_geom.wkt')
CFile.str_2_file(geom_native_wkt, native_geom_filepath)
projection = xml_obj.get_element_text_by_xpath_one(
'/TileMetadata/SpatialReference/PRJ')
if (projection is not None) and (not CUtils.equal_ignore_case(
projection, '')):
source_projection = osr.SpatialReference(wkt=projection)
source = source_projection.GetAttrValue('GEOGCS', 0) # 坐标系名称
prosrs = osr.SpatialReference()
if prosrs.ImportFromWkt(projection) == gdal.CE_None:
proj_wkt = prosrs.ExportToWkt()
native_wkt = proj_wkt
else:
native_wkt = projection
native_proj4 = prosrs.ExportToProj4()
spatial = None
rb = (0, 0)
lu = (0, 0)
geosrs = prosrs.CloneGeogCS()
ct = osr.CreateCoordinateTransformation(prosrs, geosrs)
if ct is not None:
rb = ct.TransformPoint(native_max_x, native_max_y)
lu = ct.TransformPoint(native_min_x, native_min_y)
wgs84_min_x = lu[0]
wgs84_max_y = lu[1]
wgs84_max_x = rb[0]
wgs84_min_y = rb[1]
dict_wgs84 = {
'max_x': CUtils.any_2_str(wgs84_max_x),
'max_y': CUtils.any_2_str(wgs84_max_y),
'min_x': CUtils.any_2_str(wgs84_min_x),
'min_y': CUtils.any_2_str(wgs84_min_y)
}
# 中心坐标
center_x = (wgs84_max_x - wgs84_min_x) / 2 + wgs84_min_x
center_y = (wgs84_max_y - wgs84_min_y) / 2 + wgs84_min_y
wgs84_center_wkt = 'POINT({0} {1})'.format(
center_x, center_y)
# 外边框、外包框
wgs84_bbox_wkt = CUtils.replace_placeholder(
wkt_info[:], dict_wgs84)
wgs84_geom_wkt = wgs84_bbox_wkt
wgs84_center_filepath = CFile.join_file(
file_path, file_main_name + '_wgs84_center.wkt')
CFile.str_2_file(wgs84_center_wkt, wgs84_center_filepath)
wgs84_bbox_filepath = CFile.join_file(
file_path, file_main_name + '_wgs84_bbox.wkt')
CFile.str_2_file(wgs84_bbox_wkt, wgs84_bbox_filepath)
wgs84_geom_filepath = CFile.join_file(
file_path, file_main_name + '_wgs84_geom.wkt')
CFile.str_2_file(wgs84_geom_wkt, wgs84_geom_filepath)
# </editor-fold>
# <editor-fold desc="2.投影信息">
native_source = CResource.Prj_Source_Data
# 坐标系
native_coordinate = None
# 3度带/6度带
native_degree = None
# 投影方式
native_project = None
# 带号
native_zone = None
# 创建SpatialReference对象，导入wkt信息
spatial_ref = osr.SpatialReference(wkt=native_wkt)
if spatial_ref.IsProjected():
native_project = spatial_ref.GetAttrValue('PROJECTION')
native_coordinate = spatial_ref.GetAttrValue('GEOGCS')
# native_degree = spatial_ref.GetAttrValue('GEOGCS|UNIT', 1)
native_degree, native_zone = self.get_prj_degree_zone(
spatial_ref)
elif spatial_ref.IsGeocentric():
native_project = None
native_coordinate = spatial_ref.GetAttrValue('GEOGCS')
native_degree = None
native_zone = None
# </editor-fold>
result = CResult.merge_result(self.Success, '处理完毕!')
result = CResult.merge_result_info(result, self.Name_Prj_Wkt,
native_wkt)
result = CResult.merge_result_info(result, self.Name_Prj_Proj4,
native_proj4)
result = CResult.merge_result_info(result,
self.Name_Prj_Project,
native_project)
result = CResult.merge_result_info(result,
self.Name_Prj_Coordinate,
native_coordinate)
result = CResult.merge_result_info(result,
self.Name_Prj_Source,
native_source)
result = CResult.merge_result_info(result, self.Name_Prj_Zone,
native_zone)
result = CResult.merge_result_info(result,
self.Name_Prj_Degree,
native_degree)
return result
# return CResult.merge_result(self.Success, '处理完毕!')
else:
return CResult.merge_result(self.Success, '处理完毕!')
except Exception as error:
CLogger().warning('矢量数据的空间信息处理出现异常, 错误信息为: {0}'.format(
error.__str__()))
return CResult.merge_result(
self.Failure,
'矢量数据的空间信息处理出现异常,错误信息为：{0}!'.format(error.__str__()))
def replace_placeholder(self, text: str, safe: bool = True):
return CUtils.replace_placeholder(CUtils.any_2_str(text), self.__params, safe)
def __prepare_update_data(self) -> list:
sql_list = []
temp_helper_code_list = []
sql_update_set = ''
sql_update_params = dict()
for column_index in range(self.__column_list.size()):
column = self.__column_list.column_by_index(column_index)
if column.is_primary_key or (column.value is None):
continue
try:
column_type = self.__database.db_column_type_by_name(
column.db_column_type)
column_value_type = CUtils.dict_value_by_name(
column.value, self.Name_Type, self.DataValueType_SQL)
column_value_as_text = CUtils.any_2_str(
CUtils.dict_value_by_name(column.value, self.Name_Text,
''))
# 如果值为原生sql, 则不管字段类型为何值, 都直接把sql存入insert_data语句中
if CUtils.equal_ignore_case(column_value_type,
self.DataValueType_SQL):
column_update_set = '{0}={1}'.format(
column.name, column_value_as_text)
elif CUtils.equal_ignore_case(column_value_type,
self.DataValueType_File):
column_update_set = '{0}={1}'.format(
column.name,
':{0}'.format(self.__param_name(column.name)))
self.__database.file2param(sql_update_params,
self.__param_name(column.name),
column_value_as_text)
else:
if CUtils.equal_ignore_case(
column_type.set_value_method,
self.DB_Column_Set_Method_Function):
if len(column_value_as_text
) > column_type.function_param_max_size >= 0:
column_data_id = CUtils.one_id()
temp_helper_code_list.append(column_data_id)
sql_exchange = '''
insert into ro_global_spatialhandle(code, data) values(:code, :data)
'''
param_exchange = {
'code': column_data_id,
'data': column_value_as_text
}
sql_list.append((sql_exchange, param_exchange))
column_update_set = '{0}={1}'.format(
column.name,
CUtils.replace_placeholder(
column_type.set_value_template,
dict({
self.Name_Value:
"(select data from ro_global_spatialhandle where code = '{0}')"
.format(column_data_id)
})))
else:
if column_type.function_param_quoted:
column_value_as_text = CUtils.quote(
column_value_as_text)
column_update_set = '{0}={1}'.format(
column.name,
CUtils.replace_placeholder(
column_type.set_value_template,
dict({
self.Name_Value: column_value_as_text
})))
elif CUtils.equal_ignore_case(
column_type.set_value_method,
self.DB_Column_Set_Method_Geometry):
if len(column_value_as_text
) > column_type.function_param_max_size >= 0:
column_data_id = CUtils.one_id()
temp_helper_code_list.append(column_data_id)
sql_exchange = '''
insert into ro_global_spatialhandle(code, data) values(:code, :data)
'''
param_exchange = {
'code': column_data_id,
'data': column_value_as_text
}
sql_list.append((sql_exchange, param_exchange))
column_update_set = '{0}={1}'.format(
column.name,
CUtils.replace_placeholder(
column_type.set_value_template,
dict({
self.Name_Value:
"(select data from ro_global_spatialhandle where code = '{0}')"
.format(column_data_id),
self.Name_Srid:
CUtils.dict_value_by_name(
column.value, self.Name_Srid,
settings.application.xpath_one(
self.Path_Setting_Spatial_Srid,
self.SRID_WGS84))
})))
else:
if column_type.function_param_quoted:
column_value_as_text = CUtils.quote(
column_value_as_text)
column_update_set = '{0}={1}'.format(
column.name,
CUtils.replace_placeholder(
column_type.set_value_template,
dict({
self.Name_Value:
column_value_as_text,
self.Name_Srid:
CUtils.dict_value_by_name(
column.value, self.Name_Srid,
settings.application.xpath_one(
self.Path_Setting_Spatial_Srid,
self.SRID_WGS84))
})))
else: # if CUtils.equal_ignore_case(column_type.set_value_method, self.DB_Column_Set_Method_Param):
column_update_set = '{0}={1}'.format(
column.name,
':{0}'.format(self.__param_name(column.name)))
sql_update_params[self.__param_name(
column.name)] = column_value_as_text
sql_update_set = CUtils.str_append(sql_update_set,
column_update_set, ', ')
except Exception as error:
print(error.__str__())
raise
sql_where = ''
for column_index in range(self.__column_list.size()):
column = self.__column_list.column_by_index(column_index)
if column.is_primary_key:
sql_where, sql_update_params = self.__prepare_where_condition(
column, sql_where, sql_update_params)
if not CUtils.equal_ignore_case(sql_where, ''):
sql_where = CUtils.str_append(' where ', sql_where, ' ')
sql_update = 'update {0} set {1} {2}'.format(self.__table_name,
sql_update_set, sql_where)
sql_list.append((sql_update, sql_update_params))
for temp_helper_code in temp_helper_code_list:
sql_list.append(
("delete from ro_global_spatialhandle where code = '{0}'".
format(temp_helper_code), None))
return sql_list
def get_dest_storage(self, ib_batch_no, need_storage_size, ib_option,
ib_schema, dataset_xml):
"""
计算哪一个存储最合适, 以及入库后的子目录, 可以考虑如下因素:
1. 批次
1. 所需要的存储量
1. 业务数据集内的属性
:param ib_option:
:param need_storage_size: 所需的存储量
:param ib_batch_no: 批次编号
:param ib_schema: 入库模式
:param dataset_xml: 业务数据集的xml
:return:
dest_ib_storage_id: 目标入库的存储标识, 如果没有满足要求的, 则返回None
dest_ib_root_path: 目标入库的存储的目录
desc_ib_dir_id: 入库的父目录的标识, 默认可以等于目标存储的标识
dest_ib_subpath: 入库的数据的子目录, 如果没有满足要求的, 则返回None
message: 错误消息
"""
storage_id = ''
storage_title = ''
storage_root_path = ''
storage_volumn_free = 0
storage_volumn_max = 0
ib_schema_storage_type = CJson.dict_attr_by_path(
ib_schema, 'storage.type', self.InBound_Storage_Match_Mode_Auto)
if CUtils.equal_ignore_case(ib_schema_storage_type,
self.InBound_Storage_Match_Mode_Set):
storage_id = CJson.dict_attr_by_path(ib_schema, 'storage.id', '')
if not CUtils.equal_ignore_case(storage_id, ''):
sql_storage = '''
select dm2_storage.dstid, dm2_storage.dsttitle,
coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) as root_path,
dm2_storage.dst_volumn_max, dm2_storage.dst_volumn_max - coalesce(stat.file_size_sum, 0) as free_space
from dm2_storage left join (
select dsfstorageid, sum(dsffilesize) as file_size_sum
from dm2_storage_file left join dm2_storage
on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where dm2_storage.dsttype = 'core'
group by dsfstorageid
) stat on dm2_storage.dstid = stat.dsfstorageid
where dm2_storage.dsttype = 'core' and dm2_storage.dstid = :storage_id
'''
ds_available_storage = CFactory().give_me_db(
self.get_mission_db_id()).one_row(
sql_storage, {'storage_id': storage_id})
if ds_available_storage.is_empty():
return None, None, None, None, '没有找到编号为[{0}]的存储! '.format(
storage_id)
else:
storage_volumn_max = ds_available_storage.value_by_name(
0, 'dst_volumn_max', 0)
storage_volumn_free = ds_available_storage.value_by_name(
0, 'free_space', 0)
storage_title = ds_available_storage.value_by_name(
0, 'dsttitle', '')
if storage_volumn_max > 0 and storage_volumn_free < need_storage_size:
CLogger().debug(
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 本批数据无法入库到该存储下!'
.format(storage_title, storage_volumn_free,
need_storage_size))
return \
None, None, None, None, \
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 本批数据无法入库到该存储下!'.format(
storage_title, storage_volumn_free, need_storage_size
)
storage_root_path = ds_available_storage.value_by_name(
0, 'root_path', '')
if CUtils.equal_ignore_case(storage_id, ''):
sql_available_storage = '''
select dm2_storage.dstid, dm2_storage.dsttitle,
coalesce(dm2_storage.dstownerpath, dm2_storage.dstunipath) as root_path,
dm2_storage.dst_volumn_max, dm2_storage.dst_volumn_max - coalesce(stat.file_size_sum, 0) as free_space
from dm2_storage left join (
select dsfstorageid, sum(dsffilesize) as file_size_sum
from dm2_storage_file left join dm2_storage
on dm2_storage_file.dsfstorageid = dm2_storage.dstid
where dm2_storage.dsttype = 'core'
group by dsfstorageid
) stat on dm2_storage.dstid = stat.dsfstorageid
where dm2_storage.dsttype = 'core'
order by dm2_storage.dst_volumn_max desc
'''
ds_available_storage = CFactory().give_me_db(
self.get_mission_db_id()).all_row(sql_available_storage)
for storage_index in range(0, ds_available_storage.size()):
storage_title = ds_available_storage.value_by_name(
storage_index, 'dsttitle', '')
storage_volumn_max = ds_available_storage.value_by_name(
storage_index, 'dst_volumn_max', 0)
storage_volumn_free = ds_available_storage.value_by_name(
storage_index, 'free_space', 0)
if storage_volumn_max <= 0:
storage_id = ds_available_storage.value_by_name(
storage_index, 'dstid', '')
storage_root_path = ds_available_storage.value_by_name(
storage_index, 'root_path', '')
CLogger().debug('存储[{0}]的存储空间没有限制, 系统将把本批数据入库到该存储下'.format(
storage_title))
break
elif storage_volumn_free > need_storage_size:
storage_id = ds_available_storage.value_by_name(
storage_index, 'dstid', '')
storage_root_path = ds_available_storage.value_by_name(
storage_index, 'root_path', '')
CLogger().debug(
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 系统将把本批数据入库到该存储下'
.format(storage_title, storage_volumn_free,
need_storage_size))
break
if CUtils.equal_ignore_case(storage_id, ''):
return \
None, None, None, None, \
'本批待入库数据的容量为[{0}], 未找到符合要求的存储, 请检查各个存储的剩余空间!'.format(need_storage_size)
# 当前字典将用于计算入库到存储中的相对子目录
params = dict()
# 将批次信息加入字典
params['batch_id'] = ib_batch_no
# 将待入库数据集的元数据信息(一级元素)加入字典
self.metadata_bus_2_params(dataset_xml, params)
# 开始用字典重算相对子目录
try:
ib_schema_storage_path = CUtils.replace_placeholder(
CUtils.any_2_str(CJson.dict_attr_by_path(
ib_schema, 'path', '')), params, False)
except Exception as error:
return \
None, None, None, None, \
'本批待入库数据的入库目标目录无法确认, 部分信息未提供, 具体错误信息: [{0}]!'.format(error.__str__())
if storage_volumn_max <= 0:
return \
storage_id, storage_root_path, storage_id, ib_schema_storage_path, \
'存储[{0}]的存储空间无限制, 本批数据存储所需空间为[{1}], 系统将把本批数据入库到该存储下'.format(
storage_title, need_storage_size
)
else:
return \
storage_id, storage_root_path, storage_id, ib_schema_storage_path, \
'存储[{0}]的剩余存储空间为[{1}], 本批数据存储所需空间为[{2}], 系统将把本批数据入库到该存储下'.format(
storage_title, storage_volumn_free, need_storage_size
)
def process_raster(self) -> str:
try:
# file_name_with_full_path = r'D:\test\raster_test\石嘴山市.json'
# file_main_name = CFile.file_main_name(file_name_with_full_path)
# file_path = CFile.file_path(file_name_with_full_path)
json_obj = self.metadata.metadata_json()
# json_obj.load_file(file_name_with_full_path)
# <editor-fold desc="1.空间坐标信息">
wkt_info = 'POLYGON((${min_x} ${max_y},${max_x} ${max_y},' \
'${max_x} ${min_y},${min_x} ${min_y},${min_x} ${max_y}))'
# 四至坐标
native_max_x = CUtils.to_decimal(json_obj.xpath_one('boundingbox.right', None))
native_max_y = CUtils.to_decimal(json_obj.xpath_one('boundingbox.top', None))
native_min_x = CUtils.to_decimal(json_obj.xpath_one('boundingbox.left', None))
native_min_y = CUtils.to_decimal(json_obj.xpath_one('boundingbox.bottom', None))
if (native_max_x is None) or (native_max_y is None) or (native_min_x is None) or (native_min_y is None):
native_center_wkt = None
native_bbox_wkt = None
native_geom_wkt = None
else:
dict_native = {'max_x': CUtils.any_2_str(native_max_x),
'max_y': CUtils.any_2_str(native_max_y),
'min_x': CUtils.any_2_str(native_min_x),
'min_y': CUtils.any_2_str(native_min_y)}
# 中心点坐标
center_x = (native_max_x - native_min_x) / 2 + native_min_x
center_y = (native_max_y - native_min_y) / 2 + native_min_y
native_center_wkt = 'POINT({0} {1})'.format(center_x, center_y)
# 外边框、外包框
native_bbox_wkt = CUtils.replace_placeholder(wkt_info[:], dict_native)
native_geom_wkt = native_bbox_wkt
file_path = self.file_content.work_root_dir
file_main_name = self.object_name
native_center_filepath = CFile.join_file(file_path, file_main_name + '_native_center.wkt')
CFile.str_2_file(native_center_wkt, native_center_filepath)
native_bbox_filepath = CFile.join_file(file_path, file_main_name + '_native_bbox.wkt')
CFile.str_2_file(native_bbox_wkt, native_bbox_filepath)
native_geom_filepath = CFile.join_file(file_path, file_main_name + '_native_geom.wkt')
CFile.str_2_file(native_geom_wkt, native_geom_filepath)
# wgs84转换后的四至坐标
wgs84_max_x = CUtils.to_decimal(json_obj.xpath_one('wgs84.boundingbox.right', None))
wgs84_max_y = CUtils.to_decimal(json_obj.xpath_one('wgs84.boundingbox.top', None))
wgs84_min_x = CUtils.to_decimal(json_obj.xpath_one('wgs84.boundingbox.left', None))
wgs84_min_y = CUtils.to_decimal(json_obj.xpath_one('wgs84.boundingbox.bottom', None))
if (wgs84_max_x is None) or (wgs84_max_y is None) or (wgs84_min_x is None) or (wgs84_min_y is None):
wgs84_center_wkt = None
wgs84_bbox_wkt = None
else:
dict_wgs84 = {'max_x': CUtils.any_2_str(wgs84_max_x),
'max_y': CUtils.any_2_str(wgs84_max_y),
'min_x': CUtils.any_2_str(wgs84_min_x),
'min_y': CUtils.any_2_str(wgs84_min_y)}
# 中心点坐标（wgs84）
center_x = (wgs84_max_x - wgs84_min_x) / 2 + wgs84_min_x
center_y = (wgs84_max_y - wgs84_min_y) / 2 + wgs84_min_y
wgs84_center_wkt = 'POINT({0} {1})'.format(center_x, center_y)
# 外边框、外包框（wgs84)
wgs84_bbox_wkt = CUtils.replace_placeholder(wkt_info[:], dict_wgs84)
# wgs84_geom_wkt = wgs84_bbox_wkt
wgs84_geom_filepath = CFile.join_file(file_path, file_main_name + '_wgs84_geom.wkt')
# CFile.str_2_file(wgs84_geom_wkt, wgs84_geom_filepath)
object_file_path = self.file_info.file_name_with_full_path
process_bus(object_file_path, wgs84_geom_filepath, None)
wgs84_center_filepath = CFile.join_file(file_path, file_main_name + '_wgs84_center.wkt')
CFile.str_2_file(wgs84_center_wkt, wgs84_center_filepath)
wgs84_bbox_filepath = CFile.join_file(file_path, file_main_name + '_wgs84_bbox.wkt')
CFile.str_2_file(wgs84_bbox_wkt, wgs84_bbox_filepath)
# </editor-fold>
# <editor-fold desc="2.投影信息">
native_wkt = json_obj.xpath_one('coordinate.wkt', None)
native_proj4 = json_obj.xpath_one('coordinate.proj4', None)
native_source = CResource.Prj_Source_Data
# 坐标系
native_coordinate = None
# 3度带/6度带
native_degree = None
# 投影方式
native_project = None
# 带号
native_zone = None
# 创建SpatialReference对象，导入wkt信息
spatial_ref = osr.SpatialReference(wkt=native_wkt)
if spatial_ref.IsProjected():
native_coordinate = spatial_ref.GetAttrValue('GEOGCS')
native_project = spatial_ref.GetAttrValue('PROJECTION')
# native_degree = spatial_ref.GetAttrValue('GEOGCS|UNIT', 1)
# pixel_size = json_obj.xpath_one('pixelsize.width', None)
# native_zone = self.get_prj_zone(spatial_ref, pixel_size)
native_degree, native_zone = self.get_prj_degree_zone(spatial_ref)
elif spatial_ref.IsGeocentric():
native_project = None
native_coordinate = spatial_ref.GetAttrValue('GEOGCS')
native_degree = None
native_zone = None
# </editor-fold>
result = CResult.merge_result(self.Success, '处理完毕!')
result = CResult.merge_result_info(result, self.Name_Prj_Wkt, native_wkt)
result = CResult.merge_result_info(result, self.Name_Prj_Proj4, native_proj4)
result = CResult.merge_result_info(result, self.Name_Prj_Project, native_project)
result = CResult.merge_result_info(result, self.Name_Prj_Coordinate, native_coordinate)
result = CResult.merge_result_info(result, self.Name_Prj_Source, native_source)
result = CResult.merge_result_info(result, self.Name_Prj_Zone, native_zone)
result = CResult.merge_result_info(result, self.Name_Prj_Degree, native_degree)
return result
# return CResult.merge_result(self.Success, '处理完毕!')
except Exception as error:
CLogger().warning('影像数据的空间信息处理出现异常, 错误信息为: {0}'.format(error.__str__()))
return CResult.merge_result(self.Failure, '影像数据的空间信息处理出现异常,错误信息为：{0}!'.format(error.__str__()))
def test_replace_placeholder(self):
holder_text = '${0}'.format('value')
holder_text = CUtils.replace_placeholder(holder_text, {'value': 'wangxiya'})
assert holder_text == 'wangxiya'
