def __create_jstree( self, directory, disable='folders' ): """ Loads recursively all files and folders within the given folder and its subfolders and returns jstree representation of its structure. """ userdir_jstree = None jstree_paths = [] if os.path.exists( directory ) and not os.path.islink( directory ): for ( dirpath, dirnames, filenames ) in os.walk( directory ): for dirname in dirnames: dir_path = os.path.relpath( os.path.join( dirpath, dirname ), directory ) dir_path_hash = hashlib.sha1(unicodify(dir_path).encode('utf-8')).hexdigest() disabled = True if disable == 'folders' else False jstree_paths.append( jstree.Path( dir_path, dir_path_hash, { 'type': 'folder', 'state': { 'disabled': disabled }, 'li_attr': { 'full_path': dir_path } } ) ) for filename in filenames: file_path = os.path.relpath( os.path.join( dirpath, filename ), directory ) file_path_hash = hashlib.sha1(unicodify(file_path).encode('utf-8')).hexdigest() disabled = True if disable == 'files' else False jstree_paths.append( jstree.Path( file_path, file_path_hash, { 'type': 'file', 'state': { 'disabled': disabled }, 'li_attr': { 'full_path': file_path } } ) ) else: raise exceptions.ConfigDoesNotAllowException( 'The given directory does not exist.' ) userdir_jstree = jstree.JSTree( jstree_paths ) return userdir_jstree
def index( self, user_ctx: ProvidesUserContext, target: str, format: Optional[RemoteFilesFormat], recursive: Optional[bool], disable: Optional[RemoteFilesDisableMode], ) -> List[Dict[str, Any]]: """Returns a list of remote files available to the user.""" user_file_source_context = ProvidesUserFileSourcesUserContext(user_ctx) default_recursive = False default_format = RemoteFilesFormat.uri if "://" in target: uri = target elif target == RemoteFilesTarget.userdir: uri = "gxuserimport://" default_format = RemoteFilesFormat.flat default_recursive = True elif target == RemoteFilesTarget.importdir: uri = 'gximport://' default_format = RemoteFilesFormat.flat default_recursive = True elif target in [RemoteFilesTarget.ftpdir, 'ftp']: # legacy, allow both uri = 'gxftp://' default_format = RemoteFilesFormat.flat default_recursive = True else: raise exceptions.RequestParameterInvalidException( f"Invalid target parameter supplied [{target}]") if format is None: format = default_format if recursive is None: recursive = default_recursive self._file_sources.validate_uri_root( uri, user_context=user_file_source_context) file_source_path = self._file_sources.get_file_source_path(uri) file_source = file_source_path.file_source try: index = file_source.list(file_source_path.path, recursive=recursive, user_context=user_file_source_context) except exceptions.MessageException: log.warning(f"Problem listing file source path {file_source_path}", exc_info=True) raise except Exception: message = f"Problem listing file source path {file_source_path}" log.warning(message, exc_info=True) raise exceptions.InternalServerError(message) if format == RemoteFilesFormat.flat: # rip out directories, ensure sorted by path index = [i for i in index if i["class"] == "File"] index = sorted(index, key=itemgetter("path")) if format == RemoteFilesFormat.jstree: if disable is None: disable = RemoteFilesDisableMode.folders jstree_paths = [] for ent in index: path = ent["path"] path_hash = hashlib.sha1(smart_str(path)).hexdigest() if ent["class"] == "Directory": path_type = 'folder' disabled = True if disable == RemoteFilesDisableMode.folders else False else: path_type = 'file' disabled = True if disable == RemoteFilesDisableMode.files else False jstree_paths.append( jstree.Path( path, path_hash, { 'type': path_type, 'state': { 'disabled': disabled }, 'li_attr': { 'full_path': path } })) userdir_jstree = jstree.JSTree(jstree_paths) index = userdir_jstree.jsonData() return index
def index(self, trans, **kwd): """ GET /api/remote_files/ Displays remote files. :param target: target to load available datasets from, defaults to ftpdir possible values: ftpdir, userdir, importdir :type target: str :param format: requested format of data, defaults to flat possible values: flat, jstree :returns: list of available files :rtype: list """ # If set, target must be one of 'ftpdir' (default), 'userdir', 'importdir' target = kwd.get('target', 'ftpdir') user_context = ProvidesUserFileSourcesUserContext(trans) default_recursive = False default_format = "uri" if "://" in target: uri = target elif target == 'userdir': uri = "gxuserimport://" default_format = "flat" default_recursive = True elif target == 'importdir': uri = 'gximport://' default_format = "flat" default_recursive = True elif target in ['ftpdir', 'ftp']: # legacy, allow both uri = 'gxftp://' default_format = "flat" default_recursive = True else: raise exceptions.RequestParameterInvalidException( "Invalid target parameter supplied [%s]" % target) format = kwd.get('format', default_format) recursive = kwd.get('recursive', default_recursive) file_sources = self.app.file_sources file_sources.validate_uri_root(uri, user_context=user_context) file_source_path = file_sources.get_file_source_path(uri) file_source = file_source_path.file_source index = file_source.list(file_source_path.path, recursive=recursive, user_context=user_context) if format == "flat": # rip out directories, ensure sorted by path index = [i for i in index if i["class"] == "File"] index = sorted(index, key=itemgetter("path")) if format == "jstree": disable = kwd.get('disable', 'folders') jstree_paths = [] for ent in index: path = ent["path"] path_hash = hashlib.sha1(smart_str(path)).hexdigest() if ent["class"] == "Directory": path_type = 'folder' disabled = True if disable == 'folders' else False else: path_type = 'file' disabled = True if disable == 'files' else False jstree_paths.append( jstree.Path( path, path_hash, { 'type': path_type, 'state': { 'disabled': disabled }, 'li_attr': { 'full_path': path } })) userdir_jstree = jstree.JSTree(jstree_paths) index = userdir_jstree.jsonData() return index