def open(self, path, mode, cache=None, **kwargs):
if cache is None:
cache = self.cache is not None
elif cache and self.cache is None:
cache = False
path = self.abspath(path)
yield_path = kwargs.pop("_yield_path", False)
if mode == "r":
if cache:
lpath = self._cached_copy(path, None, cache=True, **kwargs)
lpath = remove_scheme(lpath)
else:
tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True)
lpath = tmp.path
self._cached_copy(path,
add_scheme(lpath, "file"),
cache=False,
**kwargs)
try:
if yield_path:
yield lpath
else:
f = open(lpath, "r")
yield f
if not f.closed:
f.close()
finally:
if not cache:
del tmp
elif mode == "w":
tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True)
lpath = tmp.path
try:
if yield_path:
yield lpath
else:
f = open(lpath, "w")
yield f
if not f.closed:
f.close()
if tmp.exists():
self._cached_copy(add_scheme(lpath, "file"),
path,
cache=cache,
**kwargs)
finally:
del tmp
else:
raise Exception("unknown mode {}, use r or w".format(mode))
def move_to_local(self, dst=None, dir_perm=None, **kwargs):
if dst:
dst = add_scheme(_local_fs.abspath(get_path(dst)), "file")
return FileSystemFileTarget.move_to(self,
dst,
dir_perm=dir_perm,
**kwargs)
def move_from_local(self, src=None, perm=None, dir_perm=None, **kwargs):
src = add_scheme(self.fs.local_fs.abspath(get_path(src)), "file")
return FileSystemFileTarget.move_from(self,
src,
perm=perm,
dir_perm=dir_perm,
**kwargs)
def copy_and_cleanup():
try:
if tmp.exists():
self._cached_copy(add_scheme(lpath, "file"),
path,
cache=cache,
**kwargs)
finally:
cleanup()
def copy_to_local(self, dst=None, perm=None, dir_perm=None, **kwargs):
if dst:
dst = add_scheme(self.fs.local_fs.abspath(get_path(dst)), "file")
dst = FileSystemFileTarget.copy_to(self,
dst,
perm=perm,
dir_perm=dir_perm,
**kwargs)
return remove_scheme(dst)
def open(self, path, mode, cache=None, **kwargs):
if cache is None:
cache = self.cache is not None
elif cache and self.cache is None:
cache = False
yield_path = kwargs.pop("_yield_path", False)
path = self.abspath(path)
tmp = None
if mode == "r":
if cache:
lpath = self._cached_copy(path, None, cache=True, **kwargs)
lpath = remove_scheme(lpath)
else:
tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True)
lpath = tmp.path
self._cached_copy(path,
add_scheme(lpath, "file"),
cache=False,
**kwargs)
def cleanup():
if not cache and tmp.exists():
tmp.remove()
f = lpath if yield_path else open(lpath, "r")
return RemoteFileProxy(f, success_fn=cleanup, failure_fn=cleanup)
elif mode == "w":
tmp = LocalFileTarget(is_tmp=self.ext(path, n=0) or True)
lpath = tmp.path
def cleanup():
if tmp.exists():
tmp.remove()
def copy_and_cleanup():
try:
if tmp.exists():
self._cached_copy(add_scheme(lpath, "file"),
path,
cache=cache,
**kwargs)
finally:
cleanup()
f = lpath if yield_path else open(lpath, "w")
return RemoteFileProxy(f,
success_fn=copy_and_cleanup,
failure_fn=cleanup)
else:
raise Exception("unknown mode {}, use r or w".format(mode))
def _cached_copy(self,
src,
dst,
cache=None,
prefer_cache=False,
validate=None,
**kwargs):
cache = self._use_cache(cache)
# ensure absolute paths
src = self.abspath(src)
dst = self.abspath(dst) if dst else None
# determine the copy mode for code readability
# (remote-remote: "rr", remote-local: "rl", remote-cache: "rc", ...)
src_local = self.is_local(src)
dst_local = dst and self.is_local(dst)
mode = "rl"[src_local] + ("rl"[dst_local] if dst is not None else "c")
# disable caching when the mode is local-local, local-cache or remote-remote
if mode in ("ll", "lc", "rr"):
cache = False
# dst can be None, but in this case, caching should be enabled
if dst is None and not cache:
raise Exception(
"copy destination must not be empty when caching is disabled")
# paths including scheme and base
full_src = src if has_scheme(src) else self.gfal.url(src,
cmd="filecopy")
full_dst = None
if dst:
full_dst = dst if has_scheme(dst) else self.gfal.url(
dst, cmd="filecopy")
if cache:
kwargs_no_retries = kwargs.copy()
kwargs_no_retries["retries"] = 0
kwargs_no_retries = kwargs
# handle 3 cases: lr, rl, rc
if mode == "lr":
# strategy: copy to remote, copy to cache, sync stats
# copy to remote, no need to validate as we need the stat anyway
self._atomic_copy(src, full_dst, validate=False, **kwargs)
rstat = self.stat(dst, **kwargs_no_retries)
# remove the cache entry
if dst in self.cache:
self.cache.remove(dst)
# allocate cache space and copy to cache
lstat = _local_fs.stat(src)
self.cache.allocate(lstat.st_size)
full_cdst = add_scheme(self.cache.cache_path(dst), "file")
with self.cache.lock(dst):
self._atomic_copy(src, full_cdst, validate=False)
self.cache.touch(dst, (int(time.time()), rstat.st_mtime))
return dst
else: # rl, rc
# strategy: copy to cache when not up to date, sync stats, opt. copy to local
# build the full cache path of the src file
full_csrc = add_scheme(self.cache.cache_path(src), "file")
# if the file is cached and prefer_cache is true,
# return the cache path, no questions asked
# otherwise, check if the file is there and up to date
if not prefer_cache or src not in self.cache:
with self.cache.lock(src):
# in cache and outdated?
rstat = self.stat(src, **kwargs_no_retries)
if src in self.cache and abs(
self.cache.mtime(src) - rstat.st_mtime) > 1:
self.cache.remove(src, lock=False)
# in cache at all?
if src not in self.cache:
self.cache.allocate(rstat.st_size)
self._atomic_copy(full_src,
full_csrc,
validate=validate,
**kwargs)
self.cache.touch(
src, (int(time.time()), rstat.st_mtime))
if mode == "rl":
# copy to local without permission bits
copy_no_perm(remove_scheme(full_csrc),
remove_scheme(full_dst))
return dst
else: # rc
return full_csrc
else:
# simply copy and return the dst path
self._atomic_copy(full_src, full_dst, validate=validate, **kwargs)
return full_dst if dst_local else dst
def copy_from_local(self, src=None, dir_perm=None, **kwargs):
src = add_scheme(_local_fs.abspath(get_path(src)), "file")
return FileSystemFileTarget.copy_from(self,
src,
dir_perm=dir_perm,
**kwargs)
def prepare_input(path):
path = self.provide_input(os.path.abspath(path), postfix, c.dir,
render_variables)
path = add_scheme(
path, "file") if c.absolute_paths else os.path.basename(path)
return path
def move_to_local(self, dst=None, **kwargs):
if dst:
dst = add_scheme(self.fs.local_fs.abspath(get_path(dst)), "file")
dst = FileSystemFileTarget.move_to(self, dst, **kwargs)
return remove_scheme(dst)
def _cached_copy(self,
src,
dst,
perm=None,
cache=None,
prefer_cache=False,
validate=None,
**kwargs):
"""
When this method is called, both *src* and *dst* should refer to files.
"""
if self.cache is None:
cache = False
elif cache is None:
cache = self.use_cache
else:
cache = bool(cache)
# ensure absolute paths
src = self.abspath(src)
dst = dst and self.abspath(dst) or None
# determine the copy mode for code readability
# (remote-remote: "rr", remote-local: "rl", remote-cache: "rc", ...)
src_local = self.is_local(src)
dst_local = dst and self.is_local(dst)
mode = "rl"[src_local] + ("rl"[dst_local] if dst is not None else "c")
# disable caching when the mode is local-local, local-cache or remote-remote
if mode in ("ll", "lc", "rr"):
cache = False
# dst can be None, but in this case, caching should be enabled
if dst is None and not cache:
raise Exception(
"copy destination must not be empty when caching is disabled")
if cache:
kwargs_no_retries = kwargs.copy()
kwargs_no_retries["retries"] = 0
# handle 3 cases: lr, rl, rc
if mode == "lr":
# strategy: copy to remote, copy to cache, sync stats
# copy to remote, no need to validate as we compute the stat anyway
dst_uri = self._atomic_copy(src,
dst,
perm=perm,
validate=False,
**kwargs)
rstat = self.stat(dst, **kwargs_no_retries)
# remove the cache entry
if dst in self.cache:
self.cache.remove(dst)
# allocate cache space and copy to cache
lstat = self.local_fs.stat(src)
self.cache.allocate(lstat.st_size)
cdst_uri = add_scheme(self.cache.cache_path(dst), "file")
with self.cache.lock(dst):
self._atomic_copy(src, cdst_uri, validate=False)
self.cache.touch(dst, (int(time.time()), rstat.st_mtime))
return dst_uri
else: # rl, rc
# strategy: copy to cache when not up to date, sync stats, opt. copy to local
# build the uri to the cache path of the src file
csrc_uri = add_scheme(self.cache.cache_path(src), "file")
# if the file is cached and prefer_cache is true,
# return the cache path, no questions asked
# otherwise, check if the file is there and up to date
if not prefer_cache or src not in self.cache:
with self.cache.lock(src):
# in cache and outdated?
rstat = self.stat(src, **kwargs_no_retries)
if src in self.cache and abs(
self.cache.mtime(src) - rstat.st_mtime) > 1:
self.cache.remove(src, lock=False)
# in cache at all?
if src not in self.cache:
self.cache.allocate(rstat.st_size)
self._atomic_copy(src,
csrc_uri,
validate=validate,
**kwargs)
self.cache.touch(
src, (int(time.time()), rstat.st_mtime))
if mode == "rl":
# simply use the local_fs for copying
self.local_fs.copy(csrc_uri, dst, perm=perm)
return dst
else: # rc
return csrc_uri
else:
# simply copy and return the dst path
return self._atomic_copy(src,
dst,
perm=perm,
validate=validate,
**kwargs)
def uri(self, *args, **kwargs):
return add_scheme(self.fs.abspath(self.path), "file")
def uri(self, scheme=True, **kwargs):
uri = self.fs.abspath(self.path)
if scheme:
uri = add_scheme(uri, "file")
return uri
def uri(self, scheme=True, return_all=False, **kwargs):
uri = self.fs.abspath(self.path)
if scheme:
uri = add_scheme(uri, "file")
return [uri] if return_all else uri
def move_from_local(self, src=None, **kwargs):
src = add_scheme(self.fs.local_fs.abspath(get_path(src)), "file")
return self.move_from(src, **kwargs)
def copy_to_local(self, dst=None, **kwargs):
if dst:
dst = add_scheme(self.fs.local_fs.abspath(get_path(dst)), "file")
dst = self.copy_to(dst, **kwargs)
return remove_scheme(dst)
