def __init__(self,
fs,
root=TMP,
auto_flush=False,
max_size=-1,
dir_perm=0o0770,
file_perm=0o0660,
wait_delay=5,
max_waits=120):
object.__init__(self)
# create a unique name based on fs attributes
name = "{}_{}".format(fs.__class__.__name__,
create_hash(fs.gfal.base[0]))
# create the root dir, handle tmp
root = os.path.expandvars(os.path.expanduser(root)) or self.TMP
if not os.path.exists(root) and root == self.TMP:
tmp_dir = Config.instance().get_expanded("target", "tmp_dir")
base = tempfile.mkdtemp(dir=tmp_dir)
auto_flush = True
else:
base = os.path.join(root, name)
makedirs_perm(base, dir_perm)
# save attributes and configs
self.root = root
self.fs_ref = weakref.ref(fs)
self.base = base
self.name = name
self.auto_flush = auto_flush
self.max_size = max_size
self.dir_perm = dir_perm
self.file_perm = file_perm
self.wait_delay = wait_delay
self.max_waits = max_waits
# path to the global lock file which should guard global actions such as allocations
self._global_lock_path = self._lock_path(os.path.join(base, "global"))
# currently locked cache paths, only used to clean up broken files during cleanup
self._locked_cpaths = set()
logger.debug("created RemoteCache at '{}'".format(self.base))
def postfix_file(cls, path, postfix=None, add_hash=False):
"""
Adds a *postfix* to a file *path*, right before the first file extension in the base name.
When *add_hash* is *True*, a hash based on the full source path is added before the postfix.
Example:
.. code-block:: python
postfix_file("/path/to/file.tar.gz", "_1")
# -> "/path/to/file_1.tar.gz"
postfix_file("/path/to/file.txt", "_1", add_hash=True)
# -> "/path/to/file_dacc4374d3_1.txt"
*postfix* might also be a dictionary that maps patterns to actual postfix strings. When a
pattern matches the base name of the file, the associated postfix is applied and the path is
returned. You might want to use an ordered dictionary to control the first match.
"""
dirname, basename = os.path.split(path)
# get the actual postfix
_postfix = postfix
if isinstance(postfix, dict):
for pattern, _postfix in six.iteritems(postfix):
if fnmatch.fnmatch(basename, pattern):
break
else:
_postfix = ""
# optionally add a hash of the full path
if add_hash:
full_path = os.path.realpath(
os.path.expandvars(os.path.expanduser(path)))
_postfix = "_" + create_hash(full_path) + (_postfix or "")
# add the postfix
if _postfix:
parts = basename.split(".", 1)
parts[0] += _postfix
path = os.path.join(dirname, ".".join(parts))
return path
def hash(self, path, l=10):
return create_hash(self.__class__.__name__ + self.abspath(path))
def delegate_voms_proxy_glite(endpoint,
proxy_file=None,
stdout=None,
stderr=None,
cache=True):
"""
Delegates the voms proxy via gLite to an *endpoint*, e.g.
``grid-ce.physik.rwth-aachen.de:8443``. When *proxy_file* is *None*, it defaults to the result
of :py:func:`get_voms_proxy_file`. *stdout* and *stderr* are passed to the *Popen* constructor
for executing the ``glite-ce-delegate-proxy`` command. When *cache* is *True*, a json file is
created alongside the proxy file, which stores the delegation ids per endpoint. The next time
the exact same proxy should be delegated to the same endpoint, the cached delegation id is
returned.
"""
# get the proxy file
if not proxy_file:
proxy_file = get_voms_proxy_file()
proxy_file = os.path.expandvars(os.path.expanduser(proxy_file))
if not os.path.exists(proxy_file):
raise Exception("proxy file '{}' does not exist".format(proxy_file))
if cache:
if isinstance(cache, six.string_types):
cache_file = cache
else:
cache_file = proxy_file + "_delegation_cache.json"
def remove_cache():
try:
if os.path.exists(cache_file):
os.remove(cache_file)
except OSError:
pass
# create the hash of the proxy file content
with open(proxy_file, "r") as f:
proxy_hash = create_hash(f.read())
# already delegated?
cache_data = {}
if os.path.exists(cache_file):
with open(cache_file, "r") as f:
try:
cache_data = json.load(f)
except:
remove_cache()
# is the hash up-to-date?
if cache_data.get("hash") != proxy_hash:
remove_cache()
cache_data = {}
# proxy already delegated to that endpoint?
elif endpoint in cache_data.get("ids", []):
return str(cache_data["ids"][endpoint])
# do the actual delegation
delegation_id = uuid.uuid4().hex
cmd = ["glite-ce-delegate-proxy", "-e", endpoint, delegation_id]
code = interruptable_popen(cmd, stdout=stdout, stderr=stderr)[0]
if code != 0:
raise Exception(
"glite proxy delegation to endpoint {} failed".format(endpoint))
if cache:
# write the id back to the delegation file
cache_data["hash"] = proxy_hash
cache_data.setdefault("ids", {})[endpoint] = delegation_id
with open(cache_file, "w") as f:
json.dump(cache_data, f, indent=4)
os.chmod(cache_file, 0o0600)
return delegation_id
def hash(self):
target_hashes = "".join(target.hash for target in self._flat_target_list)
return create_hash(self.__class__.__name__ + target_hashes)
def hash(self):
return create_hash(self.uri(), to_int=True)
def cache_path(self, rpath):
basename = "{}_{}".format(create_hash(rpath), os.path.basename(rpath))
return os.path.join(self.base, basename)
