def __init__(self, read_artifact_cache, write_artifact_cache): """Either cache can be None, in which case we don't read from/write to it.""" artifact_roots = [] logs = [] def get_root_and_log(cache): if cache is not None: artifact_roots.append(cache.artifact_root) logs.append(cache.log) get_root_and_log(read_artifact_cache) get_root_and_log(write_artifact_cache) if len(artifact_roots) == 0: # Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it. artifact_root = None log = None else: artifact_root = artifact_roots[0] log = logs[0] if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root: raise ValueError( 'Read and write artifact caches must have the same artifact root.' ) ArtifactCache.__init__(self, log, artifact_root) self._read_artifact_cache = read_artifact_cache self._write_artifact_cache = write_artifact_cache
def __init__(self, cache, pre_write_func=None, post_read_func=None): """ cache: The underlying artifact cache. """ ArtifactCache.__init__(self, cache.log, cache.artifact_root, cache.read_only) self._cache = cache self._pre_write_func = pre_write_func self._post_read_func = post_read_func
def __init__(self, artifact_caches): if not artifact_caches: raise ValueError('Must provide at least one underlying artifact cache') log = artifact_caches[0].log artifact_root = artifact_caches[0].artifact_root if any(x.artifact_root != artifact_root for x in artifact_caches): raise ValueError('Combined artifact caches must all have the same artifact root.') ArtifactCache.__init__(self, log, artifact_root) self._artifact_caches = artifact_caches
def __init__(self, log, artifact_root, cache_root, copy_fn=None): """ cache_root: The locally cached files are stored under this directory. copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that will copy cached files into the desired destination. If unspecified, a simple file copy is used. """ ArtifactCache.__init__(self, log, artifact_root) self._cache_root = cache_root self._copy_fn = copy_fn or (lambda src, rel_dst: shutil.copy( src, os.path.join(self.artifact_root, rel_dst))) safe_mkdir(self._cache_root)
def __init__(self, log, artifact_root, cache_root, copy_fn=None): """ cache_root: The locally cached files are stored under this directory. copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that will copy cached files into the desired destination. If unspecified, a simple file copy is used. """ ArtifactCache.__init__(self, log, artifact_root) self._cache_root = cache_root self._copy_fn = copy_fn or ( lambda src, rel_dst: shutil.copy(src, os.path.join(self.artifact_root, rel_dst))) safe_mkdir(self._cache_root)
def __init__(self, artifact_caches, backfill=True): """We delegate to artifact_caches, a list of ArtifactCache instances, in order. If backfill is true then we populate earlier caches that were missing an artifact, if that artifact was found in a later cache. This is useful for priming a local cache from a remote one. """ if not artifact_caches: raise ValueError('Must provide at least one underlying artifact cache') log = artifact_caches[0].log artifact_root = artifact_caches[0].artifact_root if any(x.artifact_root != artifact_root for x in artifact_caches): raise ValueError('Combined artifact caches must all have the same artifact root.') ArtifactCache.__init__(self, log, artifact_root) self._artifact_caches = artifact_caches self._backfill = backfill
def __init__(self, artifact_caches, backfill=True): """We delegate to artifact_caches, a list of ArtifactCache instances, in order. If backfill is true then we populate earlier caches that were missing an artifact, if that artifact was found in a later cache. This is useful for priming a local cache from a remote one. """ if not artifact_caches: raise ValueError('Must provide at least one underlying artifact cache') log = artifact_caches[0].log artifact_root = artifact_caches[0].artifact_root if any(x.artifact_root != artifact_root for x in artifact_caches): raise ValueError('Combined artifact caches must all have the same artifact root.') read_only = all(x.read_only for x in artifact_caches) ArtifactCache.__init__(self, log, artifact_root, read_only) self._artifact_caches = artifact_caches self._backfill = backfill
def __init__(self, log, artifact_root, cache_root, compress=True, copy_fn=None, read_only=False): """ cache_root: The locally cached files are stored under this directory. copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that will copy cached files into the desired destination. If unspecified, a simple file copy is used. """ ArtifactCache.__init__(self, log, artifact_root, read_only) self._cache_root = os.path.expanduser(cache_root) self._compress = compress def copy(src, rel_dst): dst = os.path.join(self.artifact_root, rel_dst) safe_mkdir_for(dst) shutil.copy(src, dst) self._copy_fn = copy_fn or copy safe_mkdir(self._cache_root)
def __init__(self, log, artifact_root, url_base, compress=True, read_only=False): """ url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any path under this base. compress: Whether to compress the artifacts before storing them. """ ArtifactCache.__init__(self, log, artifact_root, read_only) parsed_url = urlparse.urlparse(url_base) if parsed_url.scheme == 'http': self._ssl = False elif parsed_url.scheme == 'https': self._ssl = True else: raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS') self._timeout_secs = 4.0 self._netloc = parsed_url.netloc self._path_prefix = parsed_url.path.rstrip('/') self.compress = compress
def __init__(self, log, artifact_root, url_base, compress=True): """ url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any path under this base. compress: Whether to compress the artifacts before storing them. """ ArtifactCache.__init__(self, log, artifact_root) parsed_url = urlparse.urlparse(url_base) if parsed_url.scheme == 'http': self._ssl = False elif parsed_url.scheme == 'https': self._ssl = True else: raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS') self._timeout_secs = 4.0 self._netloc = parsed_url.netloc self._path_prefix = parsed_url.path.rstrip('/') self.compress = compress
def __init__(self, read_artifact_cache, write_artifact_cache): """Either cache can be None, in which case we don't read from/write to it.""" artifact_roots = [] logs = [] def get_root_and_log(cache): if cache is not None: artifact_roots.append(cache.artifact_root) logs.append(cache.log) get_root_and_log(read_artifact_cache) get_root_and_log(write_artifact_cache) if len(artifact_roots) == 0: # Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it. artifact_root = None log = None else: artifact_root = artifact_roots[0] log = logs[0] if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root: raise ValueError('Read and write artifact caches must have the same artifact root.') ArtifactCache.__init__(self, log, artifact_root) self._read_artifact_cache = read_artifact_cache self._write_artifact_cache = write_artifact_cache