def __init__(self, file_type, file_name_with_full_path):
self.__file_name_with_full_path = file_name_with_full_path
self.__file_name_without_path = CFile.file_name(
self.file_name_with_full_path)
self.__file_main_name = CFile.file_main_name(
self.file_name_with_full_path)
self.__file_ext = CFile.file_ext(self.file_name_with_full_path)
self.__file_path = CFile.file_path(self.file_name_with_full_path)
self.__file_main_name_with_full_path = CFile.join_file(
self.file_path, self.file_main_name)
self.__file_type = file_type
self.__file_existed = CFile.file_or_path_exist(
self.file_name_with_full_path)
if self.__file_existed:
if CFile.is_file(self.file_name_with_full_path):
self.__file_size = CFile.file_size(
self.file_name_with_full_path)
self.__file_create_time = CFile.file_create_time(
self.file_name_with_full_path)
self.__file_access_time = CFile.file_access_time(
self.file_name_with_full_path)
self.__file_modify_time = CFile.file_modify_time(
self.file_name_with_full_path)
def register_dm_modules(self):
sql_register_dm_metadata_modules_clear = '''
truncate table dm2_modules cascade
'''
sql_register_dm_metadata_modules = '''
insert into dm2_modules(dmid, dmtitle)
values (:dmid, :dmtitle)
'''
CFactory().give_me_db().execute(sql_register_dm_metadata_modules_clear)
modules_root_dir = CSys.get_metadata_data_access_modules_root_dir()
module_file_name_list = CFile.file_or_subpath_of_path(
modules_root_dir, '*.{0}'.format(self.FileExt_Py))
for module_file_name in module_file_name_list:
if CFile.is_file(
CFile.join_file(modules_root_dir, module_file_name)) and (
not (str(module_file_name)).startswith('_')):
module_name = CFile.file_main_name(module_file_name)
module_obj = CObject.create_module_instance(
CSys.get_metadata_data_access_modules_root_name(),
module_name, CResource.DB_Server_ID_Default)
module_info = module_obj.information()
CFactory().give_me_db().execute(
sql_register_dm_metadata_modules, {
'dmid':
module_name,
'dmtitle':
CUtils.dict_value_by_name(
module_info, CResource.Name_Title, module_name)
})
def parser_file_or_subpath_of_path(self, dataset, ds_id, ds_path,
ds_rule_content, inbound_id):
"""
处理目录(完整路径)下的子目录和文件
:param inbound_id: 入库标识
:param ds_rule_content:
:param dataset: 数据集
:param ds_id: 路径标识
:param ds_path: 路径全名
:return:
"""
ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '')
ignore_file_array = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_ignore_file, None)
ignore_dir_array = settings.application.xpath_one(
self.Path_Setting_MetaData_InBound_ignore_dir, None)
file_list = CFile.file_or_subpath_of_path(ds_path)
for file_name in file_list:
file_name_with_full_path = CFile.join_file(ds_path, file_name)
if CFile.is_dir(file_name_with_full_path):
CLogger().debug('在目录{0}下发现子目录: {1}'.format(ds_path, file_name))
if CUtils.list_count(ignore_dir_array, file_name) > 0:
CLogger().debug(
'子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name))
continue
path_obj = CDMPathInfo(
self.FileType_Dir, file_name_with_full_path,
dataset.value_by_name(0, 'query_storage_id', ''), None,
ds_id,
dataset.value_by_name(0, 'query_dir_parent_objid', None),
self.get_mission_db_id(), ds_rule_content)
if path_obj.white_black_valid():
path_obj.db_check_and_update(inbound_id)
else:
CLogger().info('目录[{0}]未通过黑白名单检验, 不允许入库! '.format(
file_name_with_full_path))
elif CFile.is_file(file_name_with_full_path):
if CUtils.list_count(ignore_file_array, file_name) > 0:
CLogger().debug(
'子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name))
continue
CLogger().debug('在目录{0}下发现文件: {1}'.format(ds_path, file_name))
file_obj = CDMFileInfo(
self.FileType_File, file_name_with_full_path,
dataset.value_by_name(0, 'query_storage_id', ''), None,
ds_id,
dataset.value_by_name(0, 'query_dir_parent_objid', None),
self.get_mission_db_id(), ds_rule_content)
if file_obj.white_black_valid():
file_obj.db_check_and_update(inbound_id)
else:
CLogger().info('文件[{0}]未通过黑白名单检验, 不允许入库! '.format(
file_name_with_full_path))
def __stat_object_detail_of_schema(self) -> str:
"""
将数据附属文件的统计信息入库
. 仅适用于Directory_Itself模式
:return:
"""
result_sub_dir_count, result_file_count, result_file_size_sum = CFile.stat_of_path(
self.__detail_file_path__, self.__detail_file_recurse__,
self.__detail_file_match_text__, self.__detail_file_match_type__)
query_file_relation_name = self.file_info.file_name_with_rel_path
params = dict()
file_relation_name = CFile.file_relation_path(
self.__detail_file_path__, self.file_info.root_path)
if CUtils.equal_ignore_case(query_file_relation_name,
file_relation_name):
params['dodid'] = self.object_id
else:
params['dodid'] = CUtils.one_id()
params['dodfiletype'] = self.FileType_Dir
params['dodfileext'] = None
if CFile.is_file(self.__detail_file_path__):
params['dodfiletype'] = self.FileType_File
params['dodfileext'] = CFile.file_ext(self.__detail_file_path__)
params['dodobjectid'] = self.object_id
params['dodfilename'] = CFile.unify(file_relation_name)
params['doddircount'] = result_sub_dir_count
params['dodfilecount'] = result_file_count
params['dodfilesize'] = result_file_size_sum
params['dodfilecreatetime'] = CFile.file_create_time(
self.__detail_file_path__)
params['dodfilemodifytime'] = CFile.file_modify_time(
self.__detail_file_path__)
try:
CFactory().give_me_db(self.file_info.db_server_id).execute(
'''
INSERT INTO dm2_storage_obj_detail(
dodid, dodobjectid, dodfilename, dodfileext, dodfilesize, doddircount, dodfilecount,
dodfilecreatetime, dodfilemodifytime, dodlastmodifytime, dodfiletype)
VALUES (
:dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize, :doddircount, :dodfilecount,
:dodfilecreatetime, :dodfilemodifytime, now(), :dodfiletype)
''', params)
return CResult.merge_result(self.Success, '处理完毕!')
except Exception as error:
CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
return CResult.merge_result(
self.Failure, '数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
def custom_init(self):
"""
自定义初始化
对详情文件的路径, 匹配串, 匹配类型和是否递归处理进行设置
:return:
"""
super().custom_init()
if CFile.is_file(self.file_info.file_name_with_full_path):
self.__detail_file_path__ = self.file_info.file_path
else:
self.__detail_file_path__ = self.file_info.file_name_with_full_path
self.__detail_file_recurse__ = True
self.__detail_file_match_type__ = CFile.MatchType_Common
self.__detail_file_match_text__ = '*'
self._only_stat_file = True
