def __create_jstree( self, directory, disable='folders' ):
"""
Loads recursively all files and folders within the given folder
and its subfolders and returns jstree representation
of its structure.
"""
userdir_jstree = None
jstree_paths = []
if os.path.exists( directory ) and not os.path.islink( directory ):
for ( dirpath, dirnames, filenames ) in os.walk( directory ):
for dirname in dirnames:
dir_path = os.path.relpath( os.path.join( dirpath, dirname ), directory )
dir_path_hash = hashlib.sha1(unicodify(dir_path).encode('utf-8')).hexdigest()
disabled = True if disable == 'folders' else False
jstree_paths.append( jstree.Path( dir_path, dir_path_hash, { 'type': 'folder', 'state': { 'disabled': disabled }, 'li_attr': { 'full_path': dir_path } } ) )
for filename in filenames:
file_path = os.path.relpath( os.path.join( dirpath, filename ), directory )
file_path_hash = hashlib.sha1(unicodify(file_path).encode('utf-8')).hexdigest()
disabled = True if disable == 'files' else False
jstree_paths.append( jstree.Path( file_path, file_path_hash, { 'type': 'file', 'state': { 'disabled': disabled }, 'li_attr': { 'full_path': file_path } } ) )
else:
raise exceptions.ConfigDoesNotAllowException( 'The given directory does not exist.' )
userdir_jstree = jstree.JSTree( jstree_paths )
return userdir_jstree
def index(
self,
user_ctx: ProvidesUserContext,
target: str,
format: Optional[RemoteFilesFormat],
recursive: Optional[bool],
disable: Optional[RemoteFilesDisableMode],
) -> List[Dict[str, Any]]:
"""Returns a list of remote files available to the user."""
user_file_source_context = ProvidesUserFileSourcesUserContext(user_ctx)
default_recursive = False
default_format = RemoteFilesFormat.uri
if "://" in target:
uri = target
elif target == RemoteFilesTarget.userdir:
uri = "gxuserimport://"
default_format = RemoteFilesFormat.flat
default_recursive = True
elif target == RemoteFilesTarget.importdir:
uri = 'gximport://'
default_format = RemoteFilesFormat.flat
default_recursive = True
elif target in [RemoteFilesTarget.ftpdir, 'ftp']: # legacy, allow both
uri = 'gxftp://'
default_format = RemoteFilesFormat.flat
default_recursive = True
else:
raise exceptions.RequestParameterInvalidException(
f"Invalid target parameter supplied [{target}]")
if format is None:
format = default_format
if recursive is None:
recursive = default_recursive
self._file_sources.validate_uri_root(
uri, user_context=user_file_source_context)
file_source_path = self._file_sources.get_file_source_path(uri)
file_source = file_source_path.file_source
try:
index = file_source.list(file_source_path.path,
recursive=recursive,
user_context=user_file_source_context)
except exceptions.MessageException:
log.warning(f"Problem listing file source path {file_source_path}",
exc_info=True)
raise
except Exception:
message = f"Problem listing file source path {file_source_path}"
log.warning(message, exc_info=True)
raise exceptions.InternalServerError(message)
if format == RemoteFilesFormat.flat:
# rip out directories, ensure sorted by path
index = [i for i in index if i["class"] == "File"]
index = sorted(index, key=itemgetter("path"))
if format == RemoteFilesFormat.jstree:
if disable is None:
disable = RemoteFilesDisableMode.folders
jstree_paths = []
for ent in index:
path = ent["path"]
path_hash = hashlib.sha1(smart_str(path)).hexdigest()
if ent["class"] == "Directory":
path_type = 'folder'
disabled = True if disable == RemoteFilesDisableMode.folders else False
else:
path_type = 'file'
disabled = True if disable == RemoteFilesDisableMode.files else False
jstree_paths.append(
jstree.Path(
path, path_hash, {
'type': path_type,
'state': {
'disabled': disabled
},
'li_attr': {
'full_path': path
}
}))
userdir_jstree = jstree.JSTree(jstree_paths)
index = userdir_jstree.jsonData()
return index
def index(self, trans, **kwd):
"""
GET /api/remote_files/
Displays remote files.
:param target: target to load available datasets from, defaults to ftpdir
possible values: ftpdir, userdir, importdir
:type target: str
:param format: requested format of data, defaults to flat
possible values: flat, jstree
:returns: list of available files
:rtype: list
"""
# If set, target must be one of 'ftpdir' (default), 'userdir', 'importdir'
target = kwd.get('target', 'ftpdir')
user_context = ProvidesUserFileSourcesUserContext(trans)
default_recursive = False
default_format = "uri"
if "://" in target:
uri = target
elif target == 'userdir':
uri = "gxuserimport://"
default_format = "flat"
default_recursive = True
elif target == 'importdir':
uri = 'gximport://'
default_format = "flat"
default_recursive = True
elif target in ['ftpdir', 'ftp']: # legacy, allow both
uri = 'gxftp://'
default_format = "flat"
default_recursive = True
else:
raise exceptions.RequestParameterInvalidException(
"Invalid target parameter supplied [%s]" % target)
format = kwd.get('format', default_format)
recursive = kwd.get('recursive', default_recursive)
file_sources = self.app.file_sources
file_sources.validate_uri_root(uri, user_context=user_context)
file_source_path = file_sources.get_file_source_path(uri)
file_source = file_source_path.file_source
index = file_source.list(file_source_path.path,
recursive=recursive,
user_context=user_context)
if format == "flat":
# rip out directories, ensure sorted by path
index = [i for i in index if i["class"] == "File"]
index = sorted(index, key=itemgetter("path"))
if format == "jstree":
disable = kwd.get('disable', 'folders')
jstree_paths = []
for ent in index:
path = ent["path"]
path_hash = hashlib.sha1(smart_str(path)).hexdigest()
if ent["class"] == "Directory":
path_type = 'folder'
disabled = True if disable == 'folders' else False
else:
path_type = 'file'
disabled = True if disable == 'files' else False
jstree_paths.append(
jstree.Path(
path, path_hash, {
'type': path_type,
'state': {
'disabled': disabled
},
'li_attr': {
'full_path': path
}
}))
userdir_jstree = jstree.JSTree(jstree_paths)
index = userdir_jstree.jsonData()
return index
