def open(self, path, mode, cache=None, **kwargs): if cache is None: cache = self.cache is not None elif cache and self.cache is None: cache = False path = self.abspath(path) yield_path = kwargs.pop("_yield_path", False) if mode == "r": if cache: lpath = self._cached_copy(path, None, cache=True, **kwargs) lpath = remove_scheme(lpath) else: tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True) lpath = tmp.path self._cached_copy(path, add_scheme(lpath, "file"), cache=False, **kwargs) try: if yield_path: yield lpath else: f = open(lpath, "r") yield f if not f.closed: f.close() finally: if not cache: del tmp elif mode == "w": tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True) lpath = tmp.path try: if yield_path: yield lpath else: f = open(lpath, "w") yield f if not f.closed: f.close() if tmp.exists(): self._cached_copy(add_scheme(lpath, "file"), path, cache=cache, **kwargs) finally: del tmp else: raise Exception("unknown mode {}, use r or w".format(mode))
def move_to_local(self, dst=None, dir_perm=None, **kwargs): if dst: dst = add_scheme(_local_fs.abspath(get_path(dst)), "file") return FileSystemFileTarget.move_to(self, dst, dir_perm=dir_perm, **kwargs)
def move_from_local(self, src=None, perm=None, dir_perm=None, **kwargs): src = add_scheme(self.fs.local_fs.abspath(get_path(src)), "file") return FileSystemFileTarget.move_from(self, src, perm=perm, dir_perm=dir_perm, **kwargs)
def copy_and_cleanup(): try: if tmp.exists(): self._cached_copy(add_scheme(lpath, "file"), path, cache=cache, **kwargs) finally: cleanup()
def copy_to_local(self, dst=None, perm=None, dir_perm=None, **kwargs): if dst: dst = add_scheme(self.fs.local_fs.abspath(get_path(dst)), "file") dst = FileSystemFileTarget.copy_to(self, dst, perm=perm, dir_perm=dir_perm, **kwargs) return remove_scheme(dst)
def open(self, path, mode, cache=None, **kwargs): if cache is None: cache = self.cache is not None elif cache and self.cache is None: cache = False yield_path = kwargs.pop("_yield_path", False) path = self.abspath(path) tmp = None if mode == "r": if cache: lpath = self._cached_copy(path, None, cache=True, **kwargs) lpath = remove_scheme(lpath) else: tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True) lpath = tmp.path self._cached_copy(path, add_scheme(lpath, "file"), cache=False, **kwargs) def cleanup(): if not cache and tmp.exists(): tmp.remove() f = lpath if yield_path else open(lpath, "r") return RemoteFileProxy(f, success_fn=cleanup, failure_fn=cleanup) elif mode == "w": tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True) lpath = tmp.path def cleanup(): if tmp.exists(): tmp.remove() def copy_and_cleanup(): try: if tmp.exists(): self._cached_copy(add_scheme(lpath, "file"), path, cache=cache, **kwargs) finally: cleanup() f = lpath if yield_path else open(lpath, "w") return RemoteFileProxy(f, success_fn=copy_and_cleanup, failure_fn=cleanup) else: raise Exception("unknown mode {}, use r or w".format(mode))
def _cached_copy(self, src, dst, cache=None, prefer_cache=False, validate=None, **kwargs): cache = self._use_cache(cache) # ensure absolute paths src = self.abspath(src) dst = self.abspath(dst) if dst else None # determine the copy mode for code readability # (remote-remote: "rr", remote-local: "rl", remote-cache: "rc", ...) src_local = self.is_local(src) dst_local = dst and self.is_local(dst) mode = "rl"[src_local] + ("rl"[dst_local] if dst is not None else "c") # disable caching when the mode is local-local, local-cache or remote-remote if mode in ("ll", "lc", "rr"): cache = False # dst can be None, but in this case, caching should be enabled if dst is None and not cache: raise Exception( "copy destination must not be empty when caching is disabled") # paths including scheme and base full_src = src if has_scheme(src) else self.gfal.url(src, cmd="filecopy") full_dst = None if dst: full_dst = dst if has_scheme(dst) else self.gfal.url( dst, cmd="filecopy") if cache: kwargs_no_retries = kwargs.copy() kwargs_no_retries["retries"] = 0 kwargs_no_retries = kwargs # handle 3 cases: lr, rl, rc if mode == "lr": # strategy: copy to remote, copy to cache, sync stats # copy to remote, no need to validate as we need the stat anyway self._atomic_copy(src, full_dst, validate=False, **kwargs) rstat = self.stat(dst, **kwargs_no_retries) # remove the cache entry if dst in self.cache: self.cache.remove(dst) # allocate cache space and copy to cache lstat = _local_fs.stat(src) self.cache.allocate(lstat.st_size) full_cdst = add_scheme(self.cache.cache_path(dst), "file") with self.cache.lock(dst): self._atomic_copy(src, full_cdst, validate=False) self.cache.touch(dst, (int(time.time()), rstat.st_mtime)) return dst else: # rl, rc # strategy: copy to cache when not up to date, sync stats, opt. copy to local # build the full cache path of the src file full_csrc = add_scheme(self.cache.cache_path(src), "file") # if the file is cached and prefer_cache is true, # return the cache path, no questions asked # otherwise, check if the file is there and up to date if not prefer_cache or src not in self.cache: with self.cache.lock(src): # in cache and outdated? rstat = self.stat(src, **kwargs_no_retries) if src in self.cache and abs( self.cache.mtime(src) - rstat.st_mtime) > 1: self.cache.remove(src, lock=False) # in cache at all? if src not in self.cache: self.cache.allocate(rstat.st_size) self._atomic_copy(full_src, full_csrc, validate=validate, **kwargs) self.cache.touch( src, (int(time.time()), rstat.st_mtime)) if mode == "rl": # copy to local without permission bits copy_no_perm(remove_scheme(full_csrc), remove_scheme(full_dst)) return dst else: # rc return full_csrc else: # simply copy and return the dst path self._atomic_copy(full_src, full_dst, validate=validate, **kwargs) return full_dst if dst_local else dst
def copy_from_local(self, src=None, dir_perm=None, **kwargs): src = add_scheme(_local_fs.abspath(get_path(src)), "file") return FileSystemFileTarget.copy_from(self, src, dir_perm=dir_perm, **kwargs)
def prepare_input(path): path = self.provide_input(os.path.abspath(path), postfix, c.dir, render_variables) path = add_scheme( path, "file") if c.absolute_paths else os.path.basename(path) return path
def move_to_local(self, dst=None, **kwargs): if dst: dst = add_scheme(self.fs.local_fs.abspath(get_path(dst)), "file") dst = FileSystemFileTarget.move_to(self, dst, **kwargs) return remove_scheme(dst)
def _cached_copy(self, src, dst, perm=None, cache=None, prefer_cache=False, validate=None, **kwargs): """ When this method is called, both *src* and *dst* should refer to files. """ if self.cache is None: cache = False elif cache is None: cache = self.use_cache else: cache = bool(cache) # ensure absolute paths src = self.abspath(src) dst = dst and self.abspath(dst) or None # determine the copy mode for code readability # (remote-remote: "rr", remote-local: "rl", remote-cache: "rc", ...) src_local = self.is_local(src) dst_local = dst and self.is_local(dst) mode = "rl"[src_local] + ("rl"[dst_local] if dst is not None else "c") # disable caching when the mode is local-local, local-cache or remote-remote if mode in ("ll", "lc", "rr"): cache = False # dst can be None, but in this case, caching should be enabled if dst is None and not cache: raise Exception( "copy destination must not be empty when caching is disabled") if cache: kwargs_no_retries = kwargs.copy() kwargs_no_retries["retries"] = 0 # handle 3 cases: lr, rl, rc if mode == "lr": # strategy: copy to remote, copy to cache, sync stats # copy to remote, no need to validate as we compute the stat anyway dst_uri = self._atomic_copy(src, dst, perm=perm, validate=False, **kwargs) rstat = self.stat(dst, **kwargs_no_retries) # remove the cache entry if dst in self.cache: self.cache.remove(dst) # allocate cache space and copy to cache lstat = self.local_fs.stat(src) self.cache.allocate(lstat.st_size) cdst_uri = add_scheme(self.cache.cache_path(dst), "file") with self.cache.lock(dst): self._atomic_copy(src, cdst_uri, validate=False) self.cache.touch(dst, (int(time.time()), rstat.st_mtime)) return dst_uri else: # rl, rc # strategy: copy to cache when not up to date, sync stats, opt. copy to local # build the uri to the cache path of the src file csrc_uri = add_scheme(self.cache.cache_path(src), "file") # if the file is cached and prefer_cache is true, # return the cache path, no questions asked # otherwise, check if the file is there and up to date if not prefer_cache or src not in self.cache: with self.cache.lock(src): # in cache and outdated? rstat = self.stat(src, **kwargs_no_retries) if src in self.cache and abs( self.cache.mtime(src) - rstat.st_mtime) > 1: self.cache.remove(src, lock=False) # in cache at all? if src not in self.cache: self.cache.allocate(rstat.st_size) self._atomic_copy(src, csrc_uri, validate=validate, **kwargs) self.cache.touch( src, (int(time.time()), rstat.st_mtime)) if mode == "rl": # simply use the local_fs for copying self.local_fs.copy(csrc_uri, dst, perm=perm) return dst else: # rc return csrc_uri else: # simply copy and return the dst path return self._atomic_copy(src, dst, perm=perm, validate=validate, **kwargs)
def uri(self, *args, **kwargs): return add_scheme(self.fs.abspath(self.path), "file")
def uri(self, scheme=True, **kwargs): uri = self.fs.abspath(self.path) if scheme: uri = add_scheme(uri, "file") return uri
def uri(self, scheme=True, return_all=False, **kwargs): uri = self.fs.abspath(self.path) if scheme: uri = add_scheme(uri, "file") return [uri] if return_all else uri
def move_from_local(self, src=None, **kwargs): src = add_scheme(self.fs.local_fs.abspath(get_path(src)), "file") return self.move_from(src, **kwargs)
def copy_to_local(self, dst=None, **kwargs): if dst: dst = add_scheme(self.fs.local_fs.abspath(get_path(dst)), "file") dst = self.copy_to(dst, **kwargs) return remove_scheme(dst)