def __init__(self, fs, root=TMP, auto_flush=False, max_size=-1, dir_perm=0o0770, file_perm=0o0660, wait_delay=5, max_waits=120): object.__init__(self) # create a unique name based on fs attributes name = "{}_{}".format(fs.__class__.__name__, create_hash(fs.gfal.base[0])) # create the root dir, handle tmp root = os.path.expandvars(os.path.expanduser(root)) or self.TMP if not os.path.exists(root) and root == self.TMP: tmp_dir = Config.instance().get_expanded("target", "tmp_dir") base = tempfile.mkdtemp(dir=tmp_dir) auto_flush = True else: base = os.path.join(root, name) makedirs_perm(base, dir_perm) # save attributes and configs self.root = root self.fs_ref = weakref.ref(fs) self.base = base self.name = name self.auto_flush = auto_flush self.max_size = max_size self.dir_perm = dir_perm self.file_perm = file_perm self.wait_delay = wait_delay self.max_waits = max_waits # path to the global lock file which should guard global actions such as allocations self._global_lock_path = self._lock_path(os.path.join(base, "global")) # currently locked cache paths, only used to clean up broken files during cleanup self._locked_cpaths = set() logger.debug("created RemoteCache at '{}'".format(self.base))
def postfix_file(cls, path, postfix=None, add_hash=False): """ Adds a *postfix* to a file *path*, right before the first file extension in the base name. When *add_hash* is *True*, a hash based on the full source path is added before the postfix. Example: .. code-block:: python postfix_file("/path/to/file.tar.gz", "_1") # -> "/path/to/file_1.tar.gz" postfix_file("/path/to/file.txt", "_1", add_hash=True) # -> "/path/to/file_dacc4374d3_1.txt" *postfix* might also be a dictionary that maps patterns to actual postfix strings. When a pattern matches the base name of the file, the associated postfix is applied and the path is returned. You might want to use an ordered dictionary to control the first match. """ dirname, basename = os.path.split(path) # get the actual postfix _postfix = postfix if isinstance(postfix, dict): for pattern, _postfix in six.iteritems(postfix): if fnmatch.fnmatch(basename, pattern): break else: _postfix = "" # optionally add a hash of the full path if add_hash: full_path = os.path.realpath( os.path.expandvars(os.path.expanduser(path))) _postfix = "_" + create_hash(full_path) + (_postfix or "") # add the postfix if _postfix: parts = basename.split(".", 1) parts[0] += _postfix path = os.path.join(dirname, ".".join(parts)) return path
def hash(self, path, l=10): return create_hash(self.__class__.__name__ + self.abspath(path))
def delegate_voms_proxy_glite(endpoint, proxy_file=None, stdout=None, stderr=None, cache=True): """ Delegates the voms proxy via gLite to an *endpoint*, e.g. ``grid-ce.physik.rwth-aachen.de:8443``. When *proxy_file* is *None*, it defaults to the result of :py:func:`get_voms_proxy_file`. *stdout* and *stderr* are passed to the *Popen* constructor for executing the ``glite-ce-delegate-proxy`` command. When *cache* is *True*, a json file is created alongside the proxy file, which stores the delegation ids per endpoint. The next time the exact same proxy should be delegated to the same endpoint, the cached delegation id is returned. """ # get the proxy file if not proxy_file: proxy_file = get_voms_proxy_file() proxy_file = os.path.expandvars(os.path.expanduser(proxy_file)) if not os.path.exists(proxy_file): raise Exception("proxy file '{}' does not exist".format(proxy_file)) if cache: if isinstance(cache, six.string_types): cache_file = cache else: cache_file = proxy_file + "_delegation_cache.json" def remove_cache(): try: if os.path.exists(cache_file): os.remove(cache_file) except OSError: pass # create the hash of the proxy file content with open(proxy_file, "r") as f: proxy_hash = create_hash(f.read()) # already delegated? cache_data = {} if os.path.exists(cache_file): with open(cache_file, "r") as f: try: cache_data = json.load(f) except: remove_cache() # is the hash up-to-date? if cache_data.get("hash") != proxy_hash: remove_cache() cache_data = {} # proxy already delegated to that endpoint? elif endpoint in cache_data.get("ids", []): return str(cache_data["ids"][endpoint]) # do the actual delegation delegation_id = uuid.uuid4().hex cmd = ["glite-ce-delegate-proxy", "-e", endpoint, delegation_id] code = interruptable_popen(cmd, stdout=stdout, stderr=stderr)[0] if code != 0: raise Exception( "glite proxy delegation to endpoint {} failed".format(endpoint)) if cache: # write the id back to the delegation file cache_data["hash"] = proxy_hash cache_data.setdefault("ids", {})[endpoint] = delegation_id with open(cache_file, "w") as f: json.dump(cache_data, f, indent=4) os.chmod(cache_file, 0o0600) return delegation_id
def hash(self): target_hashes = "".join(target.hash for target in self._flat_target_list) return create_hash(self.__class__.__name__ + target_hashes)
def hash(self): return create_hash(self.uri(), to_int=True)
def cache_path(self, rpath): basename = "{}_{}".format(create_hash(rpath), os.path.basename(rpath)) return os.path.join(self.base, basename)