def getpathurl(self, path, allow_none=False): path = normpath(path) credentials = "%s:%s" % (self.user, self.passwd) if credentials == ":": url = "ftp://%s%s" % (self.host.rstrip("/"), abspath(path)) else: url = "ftp://%s@%s%s" % (credentials, self.host.rstrip("/"), abspath(path)) return url
def getpathurl(self, path, allow_none=False): path = normpath(path) credentials = '%s:%s' % (self.user, self.passwd) if credentials == ':': url = 'ftp://%s%s' % (self.host.rstrip('/'), abspath(path)) else: url = 'ftp://%s@%s%s' % (credentials, self.host.rstrip('/'), abspath(path)) return url
def copydir(self, src, dst, *args, **kwargs): """ @param src: Path to the folder to be copied @param dst: Path to the folder in which to copy the folder @return: Path to the copied folder """ src = abspath(normpath(src)) dst = abspath(normpath(dst)) return self.client.file_copy(src, dst)
def movedir(self, src, dst, *args, **kwargs): """ @param src: Path to the folder to be moved @param dst: Path to the folder in which the folder will be moved @param chunk_size: if using chunk upload @return: Path to the moved folder """ src = abspath(normpath(src)) dst = abspath(normpath(dst)) return self.client.file_move(src, dst)
def rename(self, src, dst, *args, **kwargs): """ @param src: Path to the file to be renamed @param dst: Full path with the new name @raise UnsupportedError: If trying to remove the root directory @return: Path to the renamed file """ src = abspath(normpath(src)) dst = abspath(normpath(dst)) return self.client.file_move(src, dst)
def copydir(self, src, dst, *args, **kwargs): """Copy a directory to another location. :param src: Path to the folder to be copied :param dst: Path to the folder in which to copy the folder :return: Path to the copied folder """ src = abspath(normpath(src)) dst = abspath(normpath(dst)) return self.client.file_copy(src, dst)
def rename(self, src, dst, *args, **kwargs): """Rename a file of a given path. :param src: Path to the file to be renamed :param dst: Full path with the new name :raise UnsupportedError: If trying to remove the root directory :return: Path to the renamed file """ src = abspath(normpath(src)) dst = abspath(normpath(dst)) return self.client.file_move(src, dst)
def movedir(self, src, dst, *args, **kwargs): """Move a directory to another location. :param src: Path to the folder to be moved :param dst: Path to the folder in which the folder will be moved :param chunk_size: if using chunk upload :return: Path to the moved folder """ src = abspath(normpath(src)) dst = abspath(normpath(dst)) return self.client.file_move(src, dst)
def check_template(self, template_path): """Check if a template exists""" template_path = abspath(normpath(template_path)) if template_path in self.templates: return if not self.template_fs.exists(template_path): raise MissingTemplateError(template_path)
def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): """List the the files and directories under a given path. The directory contents are returned as a list of unicode paths :param path: path to the folder to list :type path: string :param wildcard: Only returns paths that match this wildcard :type wildcard: string containing a wildcard, or a callable that accepts a path and returns a boolean :param full: returns full paths (relative to the root) :type full: bool :param absolute: returns absolute paths (paths beginning with /) :type absolute: bool :param dirs_only: if True, only return directories :type dirs_only: bool :param files_only: if True, only return files :type files_only: bool :return: a list of unicode paths """ path = abspath(normpath(path)) children = self.client.children(path) return self._listdir_helper(path, children, wildcard, full, absolute, dirs_only, files_only)
def open(self, path, mode="rb", **kwargs): """Open the named file in the given mode. This method downloads the file contents into a local temporary file so that it can be worked on efficiently. Any changes made to the file are only sent back to cloud storage when the file is flushed or closed. :param path: Path to the file to be opened :param mode: In which mode to open the file :raise ResourceNotFoundError: If given path doesn't exist and 'w' is not in mode :return: RemoteFileBuffer object """ path = abspath(normpath(path)) spooled_file = SpooledTemporaryFile(mode=mode, bufsize=MAX_BUFFER) if "w" in mode: # Truncate the file if requested self.client.put_file(path, "", True) else: # Try to write to the spooled file, if path doesn't exist create it # if 'w' is in mode try: spooled_file.write(self.client.get_file(path).read()) spooled_file.seek(0, 0) except: if "w" not in mode: raise ResourceNotFoundError(path) else: self.createfile(path, True) # This will take care of closing the socket when it's done. return RemoteFileBuffer(self, path, mode, spooled_file)
def desc(self, path): """ @return: The title for the given path. """ path = abspath(normpath(path)) info = self.getinfo(path) return info["title"]
def inner_2(fs, *args, **kwargs): new_args = [] for ndx, arg in enumerate(args): if ndx < num_paths: arg = abspath(normpath(arg)) new_args.append(arg) return func(fs, *new_args, **kwargs)
def ilistdirinfo(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): self._log(DEBUG, "Listing directory (listdirinfo) %s" % path) if dirs_only and files_only: raise ValueError("dirs_only and files_only can not both be True") for item in self.tahoeutil.list(self.dircap, path): if dirs_only and item['type'] == 'filenode': continue elif files_only and item['type'] == 'dirnode': continue if wildcard is not None: if isinstance(wildcard,basestring): if not fnmatch.fnmatch(item['name'], wildcard): continue else: if not wildcard(item['name']): continue if full: item_path = relpath(pathjoin(path, item['name'])) elif absolute: item_path = abspath(pathjoin(path, item['name'])) else: item_path = item['name'] yield (item_path, item)
def removedir(self, path, recursive=False, force=False): path = abspath(normpath(path)) if not self.exists(path): raise ResourceNotFoundError(path) if self.isfile(path): raise ResourceInvalidError(path) if not force: for _checkpath in self.listdir(path): raise DirectoryNotEmptyError(path) try: if force: for rpath in self.listdir(path, full=True): try: if self.isfile(rpath): self.remove(rpath) elif self.isdir(rpath): self.removedir(rpath, force=force) except FSError: pass self.clear_dircache(dirname(path)) self.ftp.rmd(_encode(path)) except error_reply: pass if recursive: try: self.removedir(dirname(path), recursive=True) except DirectoryNotEmptyError: pass self.clear_dircache(dirname(path), path)
def walkfiles(self, path="/", wildcard=None, dir_wildcard=None, search="breadth", ignore_errors=False): if dir_wildcard is not None: # If there is a dir_wildcard, fall back to the default impl # that uses listdir(). Otherwise we run the risk of enumerating # lots of directories that will just be thrown away. for item in super(HideFS, self).walkfiles(path, wildcard, dir_wildcard, search, ignore_errors): yield item # Otherwise, the wrapped FS may provide a more efficient impl # which we can use directly. else: if wildcard is not None and not callable(wildcard): wildcard_re = re.compile(fnmatch.translate(wildcard)) wildcard = lambda fn: bool(wildcard_re.match(fn)) walk = self.wrapped_fs.walkfiles(self._encode(path), search=search, ignore_errors=ignore_errors) for filepath in walk: filepath = abspath(self._decode(filepath)) if wildcard is not None: if not wildcard(basename(filepath)): continue if self.is_hidden(filepath): continue yield filepath
def _readdir(self, path): path = abspath(normpath(path)) if self.dircache.count: cached_dirlist = self.dircache.get(path) if cached_dirlist is not None: return cached_dirlist dirlist = {} parser = FTPListDataParser() def on_line(line): if not isinstance(line, unicode): line = line.decode('utf-8') info = parser.parse_line(line) if info: info = info.__dict__ if info['name'] not in ('.', '..'): dirlist[info['name']] = info try: self.ftp.dir(_encode(path), on_line) except error_reply: pass self.dircache[path] = dirlist return dirlist
def listdirinfo(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False): self.wrapped_fs._log(DEBUG, "Listing directory (listdirinfo) %s" % path) _fixpath = self.wrapped_fs._fixpath _path = _fixpath(path) if dirs_only and files_only: raise errors.ValueError("dirs_only and files_only can not both be True") result = [] for item in self.wrapped_fs.tahoeutil.list(self.dircap, _path): if dirs_only and item['type'] == 'filenode': continue elif files_only and item['type'] == 'dirnode': continue if wildcard is not None and \ not fnmatch.fnmatch(item['name'], wildcard): continue if full: item_path = relpath(pathjoin(_path, item['name'])) elif absolute: item_path = abspath(pathjoin(_path, item['name'])) else: item_path = item['name'] cache_name = self.wrapped_fs._fixpath(u"%s/%s" % \ (path, item['name'])) self._cache_set(cache_name, 'getinfo', (), {}, (True, item)) result.append((item_path, item)) return result
def _check_path(self, path): path = normpath(path) base, fname = pathsplit(abspath(path)) dirlist = self._readdir(base) if fname and fname not in dirlist: raise ResourceNotFoundError(path) return dirlist, fname
def iter_dirs(handle): if hasattr(handle, 'seek') and handle.seekable(): handle.seek(0) with py7zr.SevenZipFile(handle) as z: for entry in z.files: if entry.is_directory: yield abspath(entry.filename)
def iter_files(handle): if hasattr(handle, 'seek') and handle.seekable(): handle.seek(0) with zipfile.ZipFile(handle) as z: for name in filter(None, z.namelist()): if not name.endswith('/'): yield abspath(name)
def component_files(fs: FS, dirpath: str) -> Iterator[str]: """ Returns the relative filepaths for files that are contained within the directory for a PDS4 component. Files in PDS4 subcomponents will have "$" in their paths (in the subcomponent directory name). """ return (abspath(filepath) for filepath in SubFS(fs, dirpath).walk.files() if "$" not in filepath)
def desc(self, path): """Get the title of a given path. :return: The title for the given path. """ path = abspath(normpath(path)) info = self.getinfo(path) return info["title"]
def resolve_template_path(self, path, app_name, base_path="/"): """Get a template path in the appropriate directory for an app""" if path.startswith('/'): return path if isinstance(app_name, Application): app = app_name else: app = self.find_app(text_type(app_name)) template_path = abspath(pathjoin(base_path, app.templates_directory, path)) return template_path
def _IdFromPath(path, pathIdMap): # if path[0] == "/": # path2 = path[1:] # result = pathIdMap.get(path, pathIdMap.get(path2, None)) result = pathIdMap.get(abspath(path), None) if result is None: info(f"Looking up {path}") from pprint import pformat info(f"Not found in {pformat(list(pathIdMap.keys()))}") return result
def getpathurl(self, path): """ @param path: path to the file for which to return the url path @param allow_none: if true, this method can return None if there is no URL form of the given path @type allow_none: bool @return: url that corresponds to the given path, if one exists """ path = abspath(normpath(path)) return self.client.media(path)
def getinfo(self, path): """ Returned information is metadata from cloud service + a few more fields with standard names for some parts of the metadata. @param path: path to the file/folder for which to return informations @return: dictionary with informations about the specific file """ path = abspath(normpath(path)) metadata = self.client.metadata(path) return self._metadata_to_info(metadata, localtime=self.localtime)
def iter_dirs(handle): if hasattr(handle, 'seek') and handle.seekable(): handle.seek(0) kwargs = {'fileobj': handle, 'mode': 'r'} else: kwargs = {'name': handle, 'mode': 'r'} with tarfile.TarFile(**kwargs) as t: for member in t.getmembers(): if member.isdir(): yield abspath(member.name)
def iter_dirs(handle): zipname = lambda n: abspath(n).rstrip('/') seen = set() root_filter = '/'.__contains__ if hasattr(handle, 'seek') and handle.seekable(): handle.seek(0) with zipfile.ZipFile(handle) as z: for name in z.namelist(): # directory defined in the zipfile if name.endswith('/'): seen.add(name) yield zipname(name) # implicit directory else: for path in filterfalse(root_filter, recursepath(name)): if path != abspath(name) and not path in seen: seen.add(path) yield zipname(path)
def getpathurl(self, path): """Get the url of a given path. :param path: path to the file for which to return the url path :param allow_none: if true, this method can return None if there is no URL form of the given path :type allow_none: bool :return: url that corresponds to the given path, if one exists """ path = abspath(normpath(path)) return self.client.media(path)
def remove(self, path): """ @param path: path to the file to be deleted @return: None if removal was successful """ path = abspath(normpath(path)) if self.is_root(path = path): raise UnsupportedError("Can't remove the root directory") if self.isdir(path = path): raise ResourceInvalidError("Specified path is a directory. " + "Please use removedir.") self.client.file_delete(path)
def get_template(self, template_path, parse=True): template_path = abspath(normpath(template_path)) template = self.templates.get(template_path, None) if template is not None: return template if self.cache.enabled: try: info = self.template_fs.getinfokeys(template_path, 'modified_time', 'st_mtime') except ResourceNotFoundError: raise MissingTemplateError(template_path) modified_time = info.get('st_mtime', None) or datetime_to_epoch( info.get('modified_time', 0)) cache_name = "%s$%s@%s" % (TEMPLATE_VERSION, template_path, modified_time) cached_template = self.cache.get(cache_name, None) if cached_template is not None: template = Template.load(cached_template) self.templates[template_path] = template return template try: source = self.template_fs.getcontents(template_path, 'rt', encoding='utf-8') except ResourceNotFoundError: raise MissingTemplateError(template_path) except Exception as e: raise BadTemplateError(template_path) try: display_path = self.template_fs.getsyspath(template_path) except NoSysPathError: display_path = template_path if self.archive is None: lib = None else: lib = self.archive.get_template_lib(template_path) template = Template(source, display_path, raw_path=template_path, lib=lib) template.parse(self) if self.cache.enabled: self.cache.set(cache_name, template.dump(self)) self.templates[template_path] = template return template
def getinfo(self, path): """Get info from cloud service. Returned information is metadata from cloud service + a few more fields with standard names for some parts of the metadata. :param path: path to the file/folder for which to return informations :return: dictionary with informations about the specific file """ path = abspath(normpath(path)) metadata = self.client.metadata(path) return self._metadata_to_info(metadata, localtime=self.localtime)
def remove(self, path): """Remove a file of a given path. :param path: path to the file to be deleted :return: None if removal was successful """ path = abspath(normpath(path)) if self.is_root(path=path): raise UnsupportedError("Can't remove the root directory") if self.isdir(path=path): raise ResourceInvalidError( "Specified path is a directory. Please use removedir.") self.client.file_delete(path)
def removedir(self, path, *args, **kwargs): """Remove a directory of a given path. :param path: path to the file to be deleted :return: None if removal was successful """ path = abspath(normpath(path)) if self.is_root(path=path): raise UnsupportedError("Can't remove the root directory") if self.isfile(path=path): raise ResourceInvalidError( "Specified path is a directory. Please use removedir.") self.client.file_delete(path)
def get_template(self, template_path, parse=True): template_path = abspath(normpath(template_path)) template = self.templates.get(template_path, None) if template is not None: return template if self.cache.enabled: try: info = self.template_fs.getdetails(template_path) except ResourceNotFound: raise MissingTemplateError(template_path) cache_name = "%s$%s@%s" % (TEMPLATE_VERSION, template_path, info.modified) cached_template = self.cache.get(cache_name, None) if cached_template is not None: template = Template.load(cached_template) self.templates[template_path] = template return template try: source = self.template_fs.gettext(template_path) except ResourceNotFound: raise MissingTemplateError(template_path) except Exception as error: raise BadTemplateError( template_path, diagnosis="failed to read text ({})".format(text_type(error)), ) try: display_path = self.template_fs.getsyspath(template_path) except NoSysPath: display_path = template_path if self.archive is None: lib = None else: lib = self.archive.get_template_lib(template_path) template = Template(source, display_path, raw_path=template_path, lib=lib) template.parse(self) if self.cache.enabled: self.cache.set(cache_name, template.dump(self)) self.templates[template_path] = template return template
def __init__(self, archive, fs, path, library=None): self.built = False try: self.cache = archive.get_cache('parser') except Exception as e: self.cache = self._default_cache self.archive = archive self.fs = fs self.path = abspath(path) self.library = library syspath = self.fs.getsyspath(path, allow_none=True) if syspath is None: self.location = self.fs.desc(path) else: self.location = syspath self._xml = None
def makedir(self, path, recursive=False, allow_recreate=False): """Create a directory of a given path. :param path: path to the folder to be created. If only the new folder is specified it will be created in the root directory :param recursive: allows recursive creation of directories :param allow_recreate: dropbox currently doesn't support allow_recreate, so if a folder exists it will :return: Id of the created directory """ if not self._checkRecursive(recursive, path): raise UnsupportedError("Recursively create specified folder.") path = abspath(normpath(path)) return self.client.file_create_folder(path)
def iterkeys(self, root="/", m=None): """Iterate over all keys beginning with the given root path.""" if m is None: m = self._map for name in iteratepath(root): try: m = m[name] except KeyError: return for (nm, subm) in m.iteritems(): if not nm: yield abspath(root) else: k = combine(root, nm) for subk in self.iterkeys(k, subm): yield subk
def iteritems(self, root="/", m=None): """Iterate over all (key,value) pairs beginning with the given root.""" root = normpath(root) if m is None: m = self._map for name in iteratepath(root): try: m = m[name] except KeyError: return for (nm, subm) in m.iteritems(): if not nm: yield (abspath(normpath(root)), subm) else: k = combine(root, nm) for (subk, subv) in self.iteritems(k, subm): yield (subk, subv)
def walkdirs(self, path="/", wildcard=None, search="breadth", ignore_errors=False): if wildcard is not None: # If there is a wildcard, fall back to the default impl # that uses listdir(). Otherwise we run the risk of enumerating # lots of directories that will just be thrown away. for item in super(HideFS, self).walkdirs(path, wildcard, search, ignore_errors): yield item # Otherwise, the wrapped FS may provide a more efficient impl # which we can use directly. else: walk = self.wrapped_fs.walkdirs(self._encode(path), search=search, ignore_errors=ignore_errors) for dirpath in walk: if self.is_hidden(dirpath): continue yield abspath(self._decode(dirpath))
def setcontents(self, path, data, *args, **kwargs): """Sets new content to remote file Method works only with existing files and sets new content to them. @param path: Path the file in which to write the new content @param contents: File contents as a string, or any object with read and seek methods @param kwargs: additional parameters like: encoding: the type of encoding to use if data is text errors: encoding errors @param chunk_size: Number of bytes to read in a chunk, if the implementation has to resort to a read copy loop @return: Path of the updated file """ path = abspath(normpath(path)) self.client.put_file(path, data, overwrite=True) return path
def clear_dircache(self, *paths): """ Clear cached directory information. :param path: Path of directory to clear cache for, or all directories if None (the default) """ if not paths: self.dircache.clear() else: dircache = self.dircache paths = [normpath(abspath(path)) for path in paths] for cached_path in dircache.keys(): for path in paths: if isbase(cached_path, path): dircache.pop(cached_path, None) break
def get_template(self, template_path): template_path = abspath(normpath(template_path)) template = self.templates.get(template_path, None) if template is not None: return template if self.cache.enabled: try: info = self.template_fs.getinfokeys(template_path, 'modified_time', 'st_mtime') except ResourceNotFoundError: raise MissingTemplateError(template_path) modified_time = info.get('st_mtime', None) or datetime_to_epoch(info.get('modified_time', 0)) cache_name = "%s$%s@%s" % (TEMPLATE_VERSION, template_path, modified_time) cached_template = self.cache.get(cache_name, None) if cached_template is not None: template = Template.load(cached_template) self.templates[template_path] = template return template try: source = self.template_fs.getcontents(template_path, 'rt', encoding='utf-8') except ResourceNotFoundError: raise MissingTemplateError(template_path) except: raise BadTemplateError(template_path) try: display_path = self.template_fs.getsyspath(template_path) except NoSysPathError: display_path = template_path if self.archive is None: lib = None else: lib = self.archive.get_template_lib(template_path) #lib = self.archive.get_lib(lib) if lib else None template = Template(source, display_path, raw_path=template_path, lib=lib) template.parse(self) if self.cache.enabled: self.cache.set(cache_name, template.dump(self)) self.templates[template_path] = template return template
def removedir(self, path, recursive=False, force=False): if self.isfile(path): raise ResourceInvalidError(path) if not self.isdir(path): if not self.isdir(dirname(path)): raise ParentDirectoryMissingError(path) raise ResourceNotFoundError(path) if abspath(path) == "/": raise RemoveRootError(path) if force: for child_path in self.listdir(path, full=True, dirs_only=True): self.removedir(child_path, force=force) for child_path in self.listdir(path, full=True, files_only=True): self.remove(child_path) elif len(self.listdir(path)) > 0: raise DirectoryNotEmptyError(path) self.client.auth.service.files() \ .delete(fileId=self._ids[path]) \ .execute() self._ids.pop(path) if recursive and len(self.listdir(dirname(path))) == 0: self.removedir(dirname(path), recursive=recursive)