def filecopy(self, src, dst, base=None, **kwargs):
if has_scheme(src):
src_uri = self.sanitize_path(src)
else:
src_uri = self.uri(src, cmd="filecopy", base=base)
if has_scheme(dst):
dst_uri = self.sanitize_path(dst)
else:
dst_uri = self.uri(dst, cmd="filecopy", base=base)
with self.context() as ctx, self.transfer_parameters(ctx) as params:
try:
logger.debug("invoking gfal2 filecopy({}, {})".format(
src_uri, dst_uri))
ctx.filecopy(params, src_uri, dst_uri)
except gfal2.GError:
e = GFALError_filecopy(src_uri, dst_uri)
# check if the operation should be retried or raised immediately
if e.reason == e.UNKNOWN:
raise e
e.reraise()
return src_uri, dst_uri
def filecopy(self, src, dst, base=None):
if has_scheme(src):
src = self.gfal_str(src)
else:
src = self.uri(src, cmd="filecopy", base=base)
if has_scheme(dst):
dst = self.gfal_str(dst)
else:
dst = self.uri(dst, cmd="filecopy", base=base)
with self.context() as ctx, self.transfer_parameters(ctx) as params:
ctx.filecopy(params, src, dst)
return src, dst
def _cached_copy(self,
src,
dst,
cache=None,
prefer_cache=False,
validate=None,
**kwargs):
cache = self._use_cache(cache)
# ensure absolute paths
src = self.abspath(src)
dst = self.abspath(dst) if dst else None
# determine the copy mode for code readability
# (remote-remote: "rr", remote-local: "rl", remote-cache: "rc", ...)
src_local = self.is_local(src)
dst_local = dst and self.is_local(dst)
mode = "rl"[src_local] + ("rl"[dst_local] if dst is not None else "c")
# disable caching when the mode is local-local, local-cache or remote-remote
if mode in ("ll", "lc", "rr"):
cache = False
# dst can be None, but in this case, caching should be enabled
if dst is None and not cache:
raise Exception(
"copy destination must not be empty when caching is disabled")
# paths including scheme and base
full_src = src if has_scheme(src) else self.gfal.url(src,
cmd="filecopy")
full_dst = None
if dst:
full_dst = dst if has_scheme(dst) else self.gfal.url(
dst, cmd="filecopy")
if cache:
kwargs_no_retries = kwargs.copy()
kwargs_no_retries["retries"] = 0
kwargs_no_retries = kwargs
# handle 3 cases: lr, rl, rc
if mode == "lr":
# strategy: copy to remote, copy to cache, sync stats
# copy to remote, no need to validate as we need the stat anyway
self._atomic_copy(src, full_dst, validate=False, **kwargs)
rstat = self.stat(dst, **kwargs_no_retries)
# remove the cache entry
if dst in self.cache:
self.cache.remove(dst)
# allocate cache space and copy to cache
lstat = _local_fs.stat(src)
self.cache.allocate(lstat.st_size)
full_cdst = add_scheme(self.cache.cache_path(dst), "file")
with self.cache.lock(dst):
self._atomic_copy(src, full_cdst, validate=False)
self.cache.touch(dst, (int(time.time()), rstat.st_mtime))
return dst
else: # rl, rc
# strategy: copy to cache when not up to date, sync stats, opt. copy to local
# build the full cache path of the src file
full_csrc = add_scheme(self.cache.cache_path(src), "file")
# if the file is cached and prefer_cache is true,
# return the cache path, no questions asked
# otherwise, check if the file is there and up to date
if not prefer_cache or src not in self.cache:
with self.cache.lock(src):
# in cache and outdated?
rstat = self.stat(src, **kwargs_no_retries)
if src in self.cache and abs(
self.cache.mtime(src) - rstat.st_mtime) > 1:
self.cache.remove(src, lock=False)
# in cache at all?
if src not in self.cache:
self.cache.allocate(rstat.st_size)
self._atomic_copy(full_src,
full_csrc,
validate=validate,
**kwargs)
self.cache.touch(
src, (int(time.time()), rstat.st_mtime))
if mode == "rl":
# copy to local without permission bits
copy_no_perm(remove_scheme(full_csrc),
remove_scheme(full_dst))
return dst
else: # rc
return full_csrc
else:
# simply copy and return the dst path
self._atomic_copy(full_src, full_dst, validate=validate, **kwargs)
return full_dst if dst_local else dst
