def parser_file_or_subpath_of_path(self, dataset, ds_id, ds_path,
ds_rule_content, inbound_id):
"""
处理目录(完整路径)下的子目录和文件
:param inbound_id: 入库标识
:param ds_rule_content:
:param dataset: 数据集
:param ds_id: 路径标识
:param ds_path: 路径全名
:return:
"""
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ignore_file_array = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_ignore_file, None)
ignore_dir_array = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_ignore_dir, None)
file_list = CFile.file_or_subpath_of_path(ds_path)
for file_name in file_list:
file_name_with_full_path = CFile.join_file(ds_path, file_name)
if CFile.is_dir(file_name_with_full_path):
CLogger().debug('在目录{0}下发现子目录: {1}'.format(ds_path, file_name))
if CUtils.list_count(ignore_dir_array, file_name) > 0:
CLogger().debug(
'子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name))
continue
path_obj = CDMPathInfo(
self.FileType_Dir, file_name_with_full_path,
dataset.value_by_name(0, 'query_storage_id', ''), None,
ds_id,
dataset.value_by_name(0, 'query_dir_parent_objid', None),
self.get_mission_db_id(), ds_rule_content)
if path_obj.white_black_valid():
path_obj.db_check_and_update(inbound_id)
else:
CLogger().info('目录[{0}]未通过黑白名单检验, 不允许入库! '.format(
file_name_with_full_path))
elif CFile.is_file(file_name_with_full_path):
if CUtils.list_count(ignore_file_array, file_name) > 0:
CLogger().debug(
'子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name))
continue
CLogger().debug('在目录{0}下发现文件: {1}'.format(ds_path, file_name))
file_obj = CDMFileInfo(
self.FileType_File, file_name_with_full_path,
dataset.value_by_name(0, 'query_storage_id', ''), None,
ds_id,
dataset.value_by_name(0, 'query_dir_parent_objid', None),
self.get_mission_db_id(), ds_rule_content)
if file_obj.white_black_valid():
file_obj.db_check_and_update(inbound_id)
else:
CLogger().info('文件[{0}]未通过黑白名单检验, 不允许入库! '.format(
file_name_with_full_path))
def search_type(self):
"""
检索插件类型
:return:
"""
listplugin = []
plugins_root_dir = CSys.get_plugins_root_dir()
plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir)
for plugins_type in plugins_type_list:
if CFile.is_dir(CFile.join_file(
plugins_root_dir, plugins_type)) and (
not (str(plugins_type)).startswith('_')):
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), plugins_type)
path = CFile.join_file(CSys.get_plugins_root_dir(),
plugins_type)
plugins_file_list = CFile.file_or_subpath_of_path(
path, '{0}_*.{1}'.format(self.Name_Plugins,
self.FileExt_Py))
for file_name_without_path in plugins_file_list:
file_main_name = CFile.file_main_name(
file_name_without_path)
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, file_main_name, None)
plugins_info = class_classified_obj.get_information()
# 获取插件的类型和名字
plugins_info["dsodid"] = '{0}'.format(
plugins_type) + CFile.unify_seperator + '{0}'.format(
file_main_name)
listplugin.append(plugins_info)
plugin_path = []
# 遍历listplugin
for i in listplugin:
file_dict = {}
# 获取当前文件工作目录
work_path = CFile.file_abs_path('.')
# 拼接通用路径
main_path = work_path + "/imetadata/business/metadata/inbound/plugins/"
# 分割插件类型和名字
list = CUtils.dict_value_by_name(i, "dsodid",
'').split(CFile.unify_seperator)
# 拼接插件所在路径
file_path = main_path + CUtils.dict_value_by_name(
i, "dsodid", '') + "." + self.FileExt_Py
# 格式化文件路径
sorted_file_path = CFile.unify(file_path)
"""
type: dir/file/layer
source: 待压缩的文件路径
target: 压缩后路径和名字(根据用户输入的压缩地址,然后拼接出完整的压缩文件)
"""
file_dict[CResource.Name_Type] = list[0]
file_dict.setdefault(CResource.Name_Source,
[]).append(sorted_file_path)
file_dict[CResource.Name_Target] = str(
CUtils.dict_value_by_name(i, "dsodtype", '')) + ".zip"
print(file_dict)
plugin_path.append(file_dict)
return plugin_path
def __inbound_object_detail_of_schema(self, list_file_fullname):
sql_detail_insert = '''
INSERT INTO dm2_storage_obj_detail(
dodid, dodobjectid, dodfilename, dodfileext, dodfilesize,
dodfilecreatetime, dodfilemodifytime,
dodlastmodifytime, dodfiletype)
VALUES (
:dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize,
:dodfilecreatetime, :dodfilemodifytime, now(),
:dodfiletype)
'''
sql_detail_insert_params_list = []
# query_storage_id = self.file_info.storage_id
query_file_relation_name = self.file_info.file_name_with_rel_path
for item_file_name_with_path in list_file_fullname:
CLogger().debug(item_file_name_with_path)
if not CFile.file_or_path_exist(item_file_name_with_path):
continue
params = dict()
file_relation_name = CFile.file_relation_path(
item_file_name_with_path, self.file_info.root_path)
if CUtils.equal_ignore_case(query_file_relation_name,
file_relation_name):
params['dodid'] = self.object_id
else:
params['dodid'] = CUtils.one_id()
# 文件类型
params['dodfiletype'] = self.FileType_File
if CFile.is_dir(item_file_name_with_path):
params['dodfiletype'] = self.FileType_Dir
params['dodobjectid'] = self.object_id
params['dodfilename'] = CFile.unify(file_relation_name)
params['dodfileext'] = CFile.file_ext(item_file_name_with_path)
params['dodfilesize'] = CFile.file_size(item_file_name_with_path)
params['dodfilecreatetime'] = CFile.file_create_time(
item_file_name_with_path)
params['dodfilemodifytime'] = CFile.file_modify_time(
item_file_name_with_path)
# params['dodstorageid'] = query_storage_id
# params['dodfilerelationname'] = CFile.file_relation_path(
# item_file_name_with_path,
# self.file_info.root_path)
sql_params_tuple = (sql_detail_insert, params)
sql_detail_insert_params_list.append(sql_params_tuple)
if len(sql_detail_insert_params_list) > 0:
try:
CFactory().give_me_db(
self.file_info.db_server_id).execute_batch(
sql_detail_insert_params_list)
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(
error.__str__()))
return CResult.merge_result(self.Failure, '处理失败!')
return CResult.merge_result(self.Success, '处理完毕!')
def register_dm_metadata_plugins(self):
sql_register_dm_metadata_plugins_clear = '''
truncate table dm2_storage_object_def cascade
'''
sql_unregister_dm_metadata_plugins = '''
delete from dm2_storage_object_def where dsodid = :dsodid
'''
sql_register_dm_metadata_plugins = '''
insert into dm2_storage_object_def(
dsodid, dsodtitle, dsodtype, dsodtypetitle, dsodtypecode, dsodgroup, dsodgrouptitle,
dsodcatalog, dsodcatalogtitle, dsod_otheroption)
values (:dsodid, :dsodtitle, :dsodtype, :dsodtypetitle, :dsodtypecode, :dsodgroup, :dsodgrouptitle,
:dsodcatalog, :dsodcatalogtitle, :dsod_otheroption)
'''
CFactory().give_me_db().execute(sql_register_dm_metadata_plugins_clear)
plugins_root_dir = CSys.get_plugins_root_dir()
plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir)
for plugins_type in plugins_type_list:
if CFile.is_dir(CFile.join_file(
plugins_root_dir, plugins_type)) and (
not (str(plugins_type)).startswith('_')):
plugins_root_package_name = '{0}.{1}'.format(
CSys.get_plugins_package_root_name(), plugins_type)
path = CFile.join_file(CSys.get_plugins_root_dir(),
plugins_type)
plugins_file_list = CFile.file_or_subpath_of_path(
path, '{0}_*.{1}'.format(self.Name_Plugins,
self.FileExt_Py))
for file_name_without_path in plugins_file_list:
file_main_name = CFile.file_main_name(
file_name_without_path)
class_classified_obj = CObject.create_plugins_instance(
plugins_root_package_name, file_main_name, None)
plugins_info = class_classified_obj.get_information()
json_obj = CJson()
json_obj.set_value_of_name(
self.Name_Is_Spatial,
CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Is_Spatial,
False))
json_obj.set_value_of_name(
self.Name_Is_DataSet,
CUtils.dict_value_by_name(
plugins_info, CPlugins.Plugins_Info_Is_Dataset,
False))
plugins_info['dsod_otheroption'] = json_obj.to_json()
print('{0}/{1}:{2}'.format(plugins_type, file_main_name,
plugins_info))
CFactory().give_me_db().execute(
sql_unregister_dm_metadata_plugins, plugins_info)
CFactory().give_me_db().execute(
sql_register_dm_metadata_plugins, plugins_info)