def __init__(self, file_type, file_name_with_full_path): self.__file_name_with_full_path = file_name_with_full_path self.__file_name_without_path = CFile.file_name( self.file_name_with_full_path) self.__file_main_name = CFile.file_main_name( self.file_name_with_full_path) self.__file_ext = CFile.file_ext(self.file_name_with_full_path) self.__file_path = CFile.file_path(self.file_name_with_full_path) self.__file_main_name_with_full_path = CFile.join_file( self.file_path, self.file_main_name) self.__file_type = file_type self.__file_existed = CFile.file_or_path_exist( self.file_name_with_full_path) if self.__file_existed: if CFile.is_file(self.file_name_with_full_path): self.__file_size = CFile.file_size( self.file_name_with_full_path) self.__file_create_time = CFile.file_create_time( self.file_name_with_full_path) self.__file_access_time = CFile.file_access_time( self.file_name_with_full_path) self.__file_modify_time = CFile.file_modify_time( self.file_name_with_full_path)
def register_dm_modules(self): sql_register_dm_metadata_modules_clear = ''' truncate table dm2_modules cascade ''' sql_register_dm_metadata_modules = ''' insert into dm2_modules(dmid, dmtitle) values (:dmid, :dmtitle) ''' CFactory().give_me_db().execute(sql_register_dm_metadata_modules_clear) modules_root_dir = CSys.get_metadata_data_access_modules_root_dir() module_file_name_list = CFile.file_or_subpath_of_path( modules_root_dir, '*.{0}'.format(self.FileExt_Py)) for module_file_name in module_file_name_list: if CFile.is_file( CFile.join_file(modules_root_dir, module_file_name)) and ( not (str(module_file_name)).startswith('_')): module_name = CFile.file_main_name(module_file_name) module_obj = CObject.create_module_instance( CSys.get_metadata_data_access_modules_root_name(), module_name, CResource.DB_Server_ID_Default) module_info = module_obj.information() CFactory().give_me_db().execute( sql_register_dm_metadata_modules, { 'dmid': module_name, 'dmtitle': CUtils.dict_value_by_name( module_info, CResource.Name_Title, module_name) })
def parser_file_or_subpath_of_path(self, dataset, ds_id, ds_path, ds_rule_content, inbound_id): """ 处理目录(完整路径)下的子目录和文件 :param inbound_id: 入库标识 :param ds_rule_content: :param dataset: 数据集 :param ds_id: 路径标识 :param ds_path: 路径全名 :return: """ ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ignore_file_array = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_ignore_file, None) ignore_dir_array = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_ignore_dir, None) file_list = CFile.file_or_subpath_of_path(ds_path) for file_name in file_list: file_name_with_full_path = CFile.join_file(ds_path, file_name) if CFile.is_dir(file_name_with_full_path): CLogger().debug('在目录{0}下发现子目录: {1}'.format(ds_path, file_name)) if CUtils.list_count(ignore_dir_array, file_name) > 0: CLogger().debug( '子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name)) continue path_obj = CDMPathInfo( self.FileType_Dir, file_name_with_full_path, dataset.value_by_name(0, 'query_storage_id', ''), None, ds_id, dataset.value_by_name(0, 'query_dir_parent_objid', None), self.get_mission_db_id(), ds_rule_content) if path_obj.white_black_valid(): path_obj.db_check_and_update(inbound_id) else: CLogger().info('目录[{0}]未通过黑白名单检验, 不允许入库! '.format( file_name_with_full_path)) elif CFile.is_file(file_name_with_full_path): if CUtils.list_count(ignore_file_array, file_name) > 0: CLogger().debug( '子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name)) continue CLogger().debug('在目录{0}下发现文件: {1}'.format(ds_path, file_name)) file_obj = CDMFileInfo( self.FileType_File, file_name_with_full_path, dataset.value_by_name(0, 'query_storage_id', ''), None, ds_id, dataset.value_by_name(0, 'query_dir_parent_objid', None), self.get_mission_db_id(), ds_rule_content) if file_obj.white_black_valid(): file_obj.db_check_and_update(inbound_id) else: CLogger().info('文件[{0}]未通过黑白名单检验, 不允许入库! '.format( file_name_with_full_path))
def __stat_object_detail_of_schema(self) -> str: """ 将数据附属文件的统计信息入库 . 仅适用于Directory_Itself模式 :return: """ result_sub_dir_count, result_file_count, result_file_size_sum = CFile.stat_of_path( self.__detail_file_path__, self.__detail_file_recurse__, self.__detail_file_match_text__, self.__detail_file_match_type__) query_file_relation_name = self.file_info.file_name_with_rel_path params = dict() file_relation_name = CFile.file_relation_path( self.__detail_file_path__, self.file_info.root_path) if CUtils.equal_ignore_case(query_file_relation_name, file_relation_name): params['dodid'] = self.object_id else: params['dodid'] = CUtils.one_id() params['dodfiletype'] = self.FileType_Dir params['dodfileext'] = None if CFile.is_file(self.__detail_file_path__): params['dodfiletype'] = self.FileType_File params['dodfileext'] = CFile.file_ext(self.__detail_file_path__) params['dodobjectid'] = self.object_id params['dodfilename'] = CFile.unify(file_relation_name) params['doddircount'] = result_sub_dir_count params['dodfilecount'] = result_file_count params['dodfilesize'] = result_file_size_sum params['dodfilecreatetime'] = CFile.file_create_time( self.__detail_file_path__) params['dodfilemodifytime'] = CFile.file_modify_time( self.__detail_file_path__) try: CFactory().give_me_db(self.file_info.db_server_id).execute( ''' INSERT INTO dm2_storage_obj_detail( dodid, dodobjectid, dodfilename, dodfileext, dodfilesize, doddircount, dodfilecount, dodfilecreatetime, dodfilemodifytime, dodlastmodifytime, dodfiletype) VALUES ( :dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize, :doddircount, :dodfilecount, :dodfilecreatetime, :dodfilemodifytime, now(), :dodfiletype) ''', params) return CResult.merge_result(self.Success, '处理完毕!') except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format(error.__str__())) return CResult.merge_result( self.Failure, '数据库处理出现异常, 错误信息为: {0}'.format(error.__str__()))
def custom_init(self): """ 自定义初始化 对详情文件的路径, 匹配串, 匹配类型和是否递归处理进行设置 :return: """ super().custom_init() if CFile.is_file(self.file_info.file_name_with_full_path): self.__detail_file_path__ = self.file_info.file_path else: self.__detail_file_path__ = self.file_info.file_name_with_full_path self.__detail_file_recurse__ = True self.__detail_file_match_type__ = CFile.MatchType_Common self.__detail_file_match_text__ = '*' self._only_stat_file = True