def __init__(self, read_artifact_cache, write_artifact_cache):
"""Either cache can be None, in which case we don't read from/write to it."""
artifact_roots = []
logs = []
def get_root_and_log(cache):
if cache is not None:
artifact_roots.append(cache.artifact_root)
logs.append(cache.log)
get_root_and_log(read_artifact_cache)
get_root_and_log(write_artifact_cache)
if len(artifact_roots) == 0:
# Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it.
artifact_root = None
log = None
else:
artifact_root = artifact_roots[0]
log = logs[0]
if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root:
raise ValueError(
'Read and write artifact caches must have the same artifact root.'
)
ArtifactCache.__init__(self, log, artifact_root)
self._read_artifact_cache = read_artifact_cache
self._write_artifact_cache = write_artifact_cache
def __init__(self, cache, pre_write_func=None, post_read_func=None): """ cache: The underlying artifact cache. """ ArtifactCache.__init__(self, cache.log, cache.artifact_root, cache.read_only) self._cache = cache self._pre_write_func = pre_write_func self._post_read_func = post_read_func
def __init__(self, artifact_caches):
if not artifact_caches:
raise ValueError('Must provide at least one underlying artifact cache')
log = artifact_caches[0].log
artifact_root = artifact_caches[0].artifact_root
if any(x.artifact_root != artifact_root for x in artifact_caches):
raise ValueError('Combined artifact caches must all have the same artifact root.')
ArtifactCache.__init__(self, log, artifact_root)
self._artifact_caches = artifact_caches
def __init__(self, cache, pre_write_func=None, post_read_func=None):
"""
cache: The underlying artifact cache.
"""
ArtifactCache.__init__(self, cache.log, cache.artifact_root,
cache.read_only)
self._cache = cache
self._pre_write_func = pre_write_func
self._post_read_func = post_read_func
def __init__(self, log, artifact_root, cache_root, copy_fn=None):
"""
cache_root: The locally cached files are stored under this directory.
copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
will copy cached files into the desired destination. If unspecified, a simple file copy is used.
"""
ArtifactCache.__init__(self, log, artifact_root)
self._cache_root = cache_root
self._copy_fn = copy_fn or (lambda src, rel_dst: shutil.copy(
src, os.path.join(self.artifact_root, rel_dst)))
safe_mkdir(self._cache_root)
def __init__(self, log, artifact_root, cache_root, copy_fn=None):
"""
cache_root: The locally cached files are stored under this directory.
copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
will copy cached files into the desired destination. If unspecified, a simple file copy is used.
"""
ArtifactCache.__init__(self, log, artifact_root)
self._cache_root = cache_root
self._copy_fn = copy_fn or (
lambda src, rel_dst: shutil.copy(src, os.path.join(self.artifact_root, rel_dst)))
safe_mkdir(self._cache_root)
def __init__(self, artifact_caches, backfill=True):
"""We delegate to artifact_caches, a list of ArtifactCache instances, in order.
If backfill is true then we populate earlier caches that were missing an artifact,
if that artifact was found in a later cache. This is useful for priming a local cache
from a remote one.
"""
if not artifact_caches:
raise ValueError('Must provide at least one underlying artifact cache')
log = artifact_caches[0].log
artifact_root = artifact_caches[0].artifact_root
if any(x.artifact_root != artifact_root for x in artifact_caches):
raise ValueError('Combined artifact caches must all have the same artifact root.')
ArtifactCache.__init__(self, log, artifact_root)
self._artifact_caches = artifact_caches
self._backfill = backfill
def __init__(self, artifact_caches, backfill=True):
"""We delegate to artifact_caches, a list of ArtifactCache instances, in order.
If backfill is true then we populate earlier caches that were missing an artifact,
if that artifact was found in a later cache. This is useful for priming a local cache
from a remote one.
"""
if not artifact_caches:
raise ValueError('Must provide at least one underlying artifact cache')
log = artifact_caches[0].log
artifact_root = artifact_caches[0].artifact_root
if any(x.artifact_root != artifact_root for x in artifact_caches):
raise ValueError('Combined artifact caches must all have the same artifact root.')
read_only = all(x.read_only for x in artifact_caches)
ArtifactCache.__init__(self, log, artifact_root, read_only)
self._artifact_caches = artifact_caches
self._backfill = backfill
def __init__(self, log, artifact_root, cache_root, compress=True, copy_fn=None, read_only=False):
"""
cache_root: The locally cached files are stored under this directory.
copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
will copy cached files into the desired destination. If unspecified, a simple file copy is used.
"""
ArtifactCache.__init__(self, log, artifact_root, read_only)
self._cache_root = os.path.expanduser(cache_root)
self._compress = compress
def copy(src, rel_dst):
dst = os.path.join(self.artifact_root, rel_dst)
safe_mkdir_for(dst)
shutil.copy(src, dst)
self._copy_fn = copy_fn or copy
safe_mkdir(self._cache_root)
def __init__(self, log, artifact_root, url_base, compress=True, read_only=False):
"""
url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
path under this base.
compress: Whether to compress the artifacts before storing them.
"""
ArtifactCache.__init__(self, log, artifact_root, read_only)
parsed_url = urlparse.urlparse(url_base)
if parsed_url.scheme == 'http':
self._ssl = False
elif parsed_url.scheme == 'https':
self._ssl = True
else:
raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS')
self._timeout_secs = 4.0
self._netloc = parsed_url.netloc
self._path_prefix = parsed_url.path.rstrip('/')
self.compress = compress
def __init__(self, log, artifact_root, url_base, compress=True):
"""
url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
path under this base.
compress: Whether to compress the artifacts before storing them.
"""
ArtifactCache.__init__(self, log, artifact_root)
parsed_url = urlparse.urlparse(url_base)
if parsed_url.scheme == 'http':
self._ssl = False
elif parsed_url.scheme == 'https':
self._ssl = True
else:
raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS')
self._timeout_secs = 4.0
self._netloc = parsed_url.netloc
self._path_prefix = parsed_url.path.rstrip('/')
self.compress = compress
def __init__(self, read_artifact_cache, write_artifact_cache):
"""Either cache can be None, in which case we don't read from/write to it."""
artifact_roots = []
logs = []
def get_root_and_log(cache):
if cache is not None:
artifact_roots.append(cache.artifact_root)
logs.append(cache.log)
get_root_and_log(read_artifact_cache)
get_root_and_log(write_artifact_cache)
if len(artifact_roots) == 0:
# Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it.
artifact_root = None
log = None
else:
artifact_root = artifact_roots[0]
log = logs[0]
if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root:
raise ValueError('Read and write artifact caches must have the same artifact root.')
ArtifactCache.__init__(self, log, artifact_root)
self._read_artifact_cache = read_artifact_cache
self._write_artifact_cache = write_artifact_cache
def __init__(self,
log,
artifact_root,
cache_root,
compress=True,
copy_fn=None,
read_only=False):
"""
cache_root: The locally cached files are stored under this directory.
copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
will copy cached files into the desired destination. If unspecified, a simple file copy is used.
"""
ArtifactCache.__init__(self, log, artifact_root, read_only)
self._cache_root = os.path.expanduser(cache_root)
self._compress = compress
def copy(src, rel_dst):
dst = os.path.join(self.artifact_root, rel_dst)
safe_mkdir_for(dst)
shutil.copy(src, dst)
self._copy_fn = copy_fn or copy
safe_mkdir(self._cache_root)