def parser_file_or_subpath_of_path(self, dataset, ds_id, ds_path, ds_rule_content, inbound_id): """ 处理目录(完整路径)下的子目录和文件 :param inbound_id: 入库标识 :param ds_rule_content: :param dataset: 数据集 :param ds_id: 路径标识 :param ds_path: 路径全名 :return: """ ds_storage_id = dataset.value_by_name(0, 'query_storage_id', '') ignore_file_array = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_ignore_file, None) ignore_dir_array = settings.application.xpath_one( self.Path_Setting_MetaData_InBound_ignore_dir, None) file_list = CFile.file_or_subpath_of_path(ds_path) for file_name in file_list: file_name_with_full_path = CFile.join_file(ds_path, file_name) if CFile.is_dir(file_name_with_full_path): CLogger().debug('在目录{0}下发现子目录: {1}'.format(ds_path, file_name)) if CUtils.list_count(ignore_dir_array, file_name) > 0: CLogger().debug( '子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name)) continue path_obj = CDMPathInfo( self.FileType_Dir, file_name_with_full_path, dataset.value_by_name(0, 'query_storage_id', ''), None, ds_id, dataset.value_by_name(0, 'query_dir_parent_objid', None), self.get_mission_db_id(), ds_rule_content) if path_obj.white_black_valid(): path_obj.db_check_and_update(inbound_id) else: CLogger().info('目录[{0}]未通过黑白名单检验, 不允许入库! '.format( file_name_with_full_path)) elif CFile.is_file(file_name_with_full_path): if CUtils.list_count(ignore_file_array, file_name) > 0: CLogger().debug( '子目录: {0}在指定的忽略入库名单中, 将不入库! '.format(file_name)) continue CLogger().debug('在目录{0}下发现文件: {1}'.format(ds_path, file_name)) file_obj = CDMFileInfo( self.FileType_File, file_name_with_full_path, dataset.value_by_name(0, 'query_storage_id', ''), None, ds_id, dataset.value_by_name(0, 'query_dir_parent_objid', None), self.get_mission_db_id(), ds_rule_content) if file_obj.white_black_valid(): file_obj.db_check_and_update(inbound_id) else: CLogger().info('文件[{0}]未通过黑白名单检验, 不允许入库! '.format( file_name_with_full_path))
def search_type(self): """ 检索插件类型 :return: """ listplugin = [] plugins_root_dir = CSys.get_plugins_root_dir() plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir) for plugins_type in plugins_type_list: if CFile.is_dir(CFile.join_file( plugins_root_dir, plugins_type)) and ( not (str(plugins_type)).startswith('_')): plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), plugins_type) path = CFile.join_file(CSys.get_plugins_root_dir(), plugins_type) plugins_file_list = CFile.file_or_subpath_of_path( path, '{0}_*.{1}'.format(self.Name_Plugins, self.FileExt_Py)) for file_name_without_path in plugins_file_list: file_main_name = CFile.file_main_name( file_name_without_path) class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, file_main_name, None) plugins_info = class_classified_obj.get_information() # 获取插件的类型和名字 plugins_info["dsodid"] = '{0}'.format( plugins_type) + CFile.unify_seperator + '{0}'.format( file_main_name) listplugin.append(plugins_info) plugin_path = [] # 遍历listplugin for i in listplugin: file_dict = {} # 获取当前文件工作目录 work_path = CFile.file_abs_path('.') # 拼接通用路径 main_path = work_path + "/imetadata/business/metadata/inbound/plugins/" # 分割插件类型和名字 list = CUtils.dict_value_by_name(i, "dsodid", '').split(CFile.unify_seperator) # 拼接插件所在路径 file_path = main_path + CUtils.dict_value_by_name( i, "dsodid", '') + "." + self.FileExt_Py # 格式化文件路径 sorted_file_path = CFile.unify(file_path) """ type: dir/file/layer source: 待压缩的文件路径 target: 压缩后路径和名字(根据用户输入的压缩地址,然后拼接出完整的压缩文件) """ file_dict[CResource.Name_Type] = list[0] file_dict.setdefault(CResource.Name_Source, []).append(sorted_file_path) file_dict[CResource.Name_Target] = str( CUtils.dict_value_by_name(i, "dsodtype", '')) + ".zip" print(file_dict) plugin_path.append(file_dict) return plugin_path
def __inbound_object_detail_of_schema(self, list_file_fullname): sql_detail_insert = ''' INSERT INTO dm2_storage_obj_detail( dodid, dodobjectid, dodfilename, dodfileext, dodfilesize, dodfilecreatetime, dodfilemodifytime, dodlastmodifytime, dodfiletype) VALUES ( :dodid, :dodobjectid, :dodfilename, :dodfileext, :dodfilesize, :dodfilecreatetime, :dodfilemodifytime, now(), :dodfiletype) ''' sql_detail_insert_params_list = [] # query_storage_id = self.file_info.storage_id query_file_relation_name = self.file_info.file_name_with_rel_path for item_file_name_with_path in list_file_fullname: CLogger().debug(item_file_name_with_path) if not CFile.file_or_path_exist(item_file_name_with_path): continue params = dict() file_relation_name = CFile.file_relation_path( item_file_name_with_path, self.file_info.root_path) if CUtils.equal_ignore_case(query_file_relation_name, file_relation_name): params['dodid'] = self.object_id else: params['dodid'] = CUtils.one_id() # 文件类型 params['dodfiletype'] = self.FileType_File if CFile.is_dir(item_file_name_with_path): params['dodfiletype'] = self.FileType_Dir params['dodobjectid'] = self.object_id params['dodfilename'] = CFile.unify(file_relation_name) params['dodfileext'] = CFile.file_ext(item_file_name_with_path) params['dodfilesize'] = CFile.file_size(item_file_name_with_path) params['dodfilecreatetime'] = CFile.file_create_time( item_file_name_with_path) params['dodfilemodifytime'] = CFile.file_modify_time( item_file_name_with_path) # params['dodstorageid'] = query_storage_id # params['dodfilerelationname'] = CFile.file_relation_path( # item_file_name_with_path, # self.file_info.root_path) sql_params_tuple = (sql_detail_insert, params) sql_detail_insert_params_list.append(sql_params_tuple) if len(sql_detail_insert_params_list) > 0: try: CFactory().give_me_db( self.file_info.db_server_id).execute_batch( sql_detail_insert_params_list) except Exception as error: CLogger().warning('数据库处理出现异常, 错误信息为: {0}'.format( error.__str__())) return CResult.merge_result(self.Failure, '处理失败!') return CResult.merge_result(self.Success, '处理完毕!')
def register_dm_metadata_plugins(self): sql_register_dm_metadata_plugins_clear = ''' truncate table dm2_storage_object_def cascade ''' sql_unregister_dm_metadata_plugins = ''' delete from dm2_storage_object_def where dsodid = :dsodid ''' sql_register_dm_metadata_plugins = ''' insert into dm2_storage_object_def( dsodid, dsodtitle, dsodtype, dsodtypetitle, dsodtypecode, dsodgroup, dsodgrouptitle, dsodcatalog, dsodcatalogtitle, dsod_otheroption) values (:dsodid, :dsodtitle, :dsodtype, :dsodtypetitle, :dsodtypecode, :dsodgroup, :dsodgrouptitle, :dsodcatalog, :dsodcatalogtitle, :dsod_otheroption) ''' CFactory().give_me_db().execute(sql_register_dm_metadata_plugins_clear) plugins_root_dir = CSys.get_plugins_root_dir() plugins_type_list = CFile.file_or_subpath_of_path(plugins_root_dir) for plugins_type in plugins_type_list: if CFile.is_dir(CFile.join_file( plugins_root_dir, plugins_type)) and ( not (str(plugins_type)).startswith('_')): plugins_root_package_name = '{0}.{1}'.format( CSys.get_plugins_package_root_name(), plugins_type) path = CFile.join_file(CSys.get_plugins_root_dir(), plugins_type) plugins_file_list = CFile.file_or_subpath_of_path( path, '{0}_*.{1}'.format(self.Name_Plugins, self.FileExt_Py)) for file_name_without_path in plugins_file_list: file_main_name = CFile.file_main_name( file_name_without_path) class_classified_obj = CObject.create_plugins_instance( plugins_root_package_name, file_main_name, None) plugins_info = class_classified_obj.get_information() json_obj = CJson() json_obj.set_value_of_name( self.Name_Is_Spatial, CUtils.dict_value_by_name( plugins_info, CPlugins.Plugins_Info_Is_Spatial, False)) json_obj.set_value_of_name( self.Name_Is_DataSet, CUtils.dict_value_by_name( plugins_info, CPlugins.Plugins_Info_Is_Dataset, False)) plugins_info['dsod_otheroption'] = json_obj.to_json() print('{0}/{1}:{2}'.format(plugins_type, file_main_name, plugins_info)) CFactory().give_me_db().execute( sql_unregister_dm_metadata_plugins, plugins_info) CFactory().give_me_db().execute( sql_register_dm_metadata_plugins, plugins_info)