def __init__(self, read_artifact_cache, write_artifact_cache):
"""Either cache can be None, in which case we don't read from/write to it."""
artifact_roots = []
logs = []
def get_root_and_log(cache):
if cache is not None:
artifact_roots.append(cache.artifact_root)
logs.append(cache.log)
get_root_and_log(read_artifact_cache)
get_root_and_log(write_artifact_cache)
if len(artifact_roots) == 0:
# Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it.
artifact_root = None
log = None
else:
artifact_root = artifact_roots[0]
log = logs[0]
if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root:
raise ValueError(
'Read and write artifact caches must have the same artifact root.'
)
ArtifactCache.__init__(self, log, artifact_root)
self._read_artifact_cache = read_artifact_cache
self._write_artifact_cache = write_artifact_cache
def __init__(self, artifact_caches, backfill=True):
"""We delegate to artifact_caches, a list of ArtifactCache instances, in order.
If backfill is true then we populate earlier caches that were missing an artifact,
if that artifact was found in a later cache. This is useful for priming a local cache
from a remote one.
"""
if not artifact_caches:
raise ValueError('Must provide at least one underlying artifact cache')
log = artifact_caches[0].log
artifact_root = artifact_caches[0].artifact_root
if any(x.artifact_root != artifact_root for x in artifact_caches):
raise ValueError('Combined artifact caches must all have the same artifact root.')
ArtifactCache.__init__(self, log, artifact_root)
self._artifact_caches = artifact_caches
self._backfill = backfill
def __init__(self, log, artifact_root, cache_root, compress=True, copy_fn=None):
"""
cache_root: The locally cached files are stored under this directory.
copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
will copy cached files into the desired destination. If unspecified, a simple file copy is used.
"""
ArtifactCache.__init__(self, log, artifact_root)
self._cache_root = os.path.expanduser(cache_root)
self._compress = compress
def copy(src, rel_dst):
dst = os.path.join(self.artifact_root, rel_dst)
safe_mkdir_for(dst)
shutil.copy(src, dst)
self._copy_fn = copy_fn or copy
safe_mkdir(self._cache_root)
def __init__(self, log, artifact_root, url_base, compress=True):
"""
url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
path under this base.
compress: Whether to compress the artifacts before storing them.
"""
ArtifactCache.__init__(self, log, artifact_root)
parsed_url = urlparse.urlparse(url_base)
if parsed_url.scheme == 'http':
self._ssl = False
elif parsed_url.scheme == 'https':
self._ssl = True
else:
raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS')
self._timeout_secs = 4.0
self._netloc = parsed_url.netloc
self._path_prefix = parsed_url.path.rstrip('/')
self.compress = compress
def __init__(self, read_artifact_cache, write_artifact_cache):
"""Either cache can be None, in which case we don't read from/write to it."""
artifact_roots = []
def get_root_and_log(cache):
if cache is not None:
artifact_roots.append(cache.artifact_root)
get_root_and_log(read_artifact_cache)
get_root_and_log(write_artifact_cache)
if len(artifact_roots) == 0:
# Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it.
artifact_root = None
else:
artifact_root = artifact_roots[0]
if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root:
raise ValueError('Read and write artifact caches must have the same artifact root.')
ArtifactCache.__init__(self, artifact_root)
self._read_artifact_cache = read_artifact_cache
self._write_artifact_cache = write_artifact_cache
def __init__(self, log, artifact_root, url_base, compress=True):
"""
url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
path under this base.
compress: Whether to compress the artifacts before storing them.
"""
ArtifactCache.__init__(self, log, artifact_root)
parsed_url = urlparse.urlparse(url_base)
if parsed_url.scheme == 'http':
self._ssl = False
elif parsed_url.scheme == 'https':
self._ssl = True
else:
raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS')
self._timeout_secs = 4.0
self._netloc = parsed_url.netloc
self._path_prefix = parsed_url.path.rstrip(b'/')
self.compress = compress
# To enable connection reuse, all requests must be created from same session.
# TODO: Re-evaluate session's life-cycle if/when a longer-lived pants process exists.
self._session = requests.Session()
def __init__(self,
log,
artifact_root,
cache_root,
compress=True,
copy_fn=None):
"""
cache_root: The locally cached files are stored under this directory.
copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
will copy cached files into the desired destination. If unspecified, a simple file copy is used.
"""
ArtifactCache.__init__(self, log, artifact_root)
self._cache_root = os.path.expanduser(cache_root)
self._compress = compress
def copy(src, rel_dst):
dst = os.path.join(self.artifact_root, rel_dst)
safe_mkdir_for(dst)
shutil.copy(src, dst)
self._copy_fn = copy_fn or copy
safe_mkdir(self._cache_root)
def __init__(self, log, artifact_root, url_base, compress=True):
"""
url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
path under this base.
compress: Whether to compress the artifacts before storing them.
"""
ArtifactCache.__init__(self, log, artifact_root)
parsed_url = urlparse.urlparse(url_base)
if parsed_url.scheme == 'http':
self._ssl = False
elif parsed_url.scheme == 'https':
self._ssl = True
else:
raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS')
self._timeout_secs = 4.0
self._netloc = parsed_url.netloc
self._path_prefix = parsed_url.path.rstrip('/')
self.compress = compress
# Reduce the somewhat verbose logging of requests.
# TODO do this in a central place
logging.getLogger('requests').setLevel(logging.WARNING)
def __init__(self, log, artifact_root, url_base, compress=True):
"""
url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
path under this base.
compress: Whether to compress the artifacts before storing them.
"""
ArtifactCache.__init__(self, log, artifact_root)
parsed_url = urlparse.urlparse(url_base)
if parsed_url.scheme == 'http':
self._ssl = False
elif parsed_url.scheme == 'https':
self._ssl = True
else:
raise ValueError(
'RESTfulArtifactCache only supports HTTP and HTTPS')
self._timeout_secs = 4.0
self._netloc = parsed_url.netloc
self._path_prefix = parsed_url.path.rstrip('/')
self.compress = compress
# Reduce the somewhat verbose logging of requests.
# TODO do this in a central place
logging.getLogger('requests').setLevel(logging.WARNING)
