def __init__(self, read_artifact_cache, write_artifact_cache): """Either cache can be None, in which case we don't read from/write to it.""" artifact_roots = [] logs = [] def get_root_and_log(cache): if cache is not None: artifact_roots.append(cache.artifact_root) logs.append(cache.log) get_root_and_log(read_artifact_cache) get_root_and_log(write_artifact_cache) if len(artifact_roots) == 0: # Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it. artifact_root = None log = None else: artifact_root = artifact_roots[0] log = logs[0] if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root: raise ValueError( 'Read and write artifact caches must have the same artifact root.' ) ArtifactCache.__init__(self, log, artifact_root) self._read_artifact_cache = read_artifact_cache self._write_artifact_cache = write_artifact_cache
def __init__(self, artifact_caches, backfill=True): """We delegate to artifact_caches, a list of ArtifactCache instances, in order. If backfill is true then we populate earlier caches that were missing an artifact, if that artifact was found in a later cache. This is useful for priming a local cache from a remote one. """ if not artifact_caches: raise ValueError('Must provide at least one underlying artifact cache') log = artifact_caches[0].log artifact_root = artifact_caches[0].artifact_root if any(x.artifact_root != artifact_root for x in artifact_caches): raise ValueError('Combined artifact caches must all have the same artifact root.') ArtifactCache.__init__(self, log, artifact_root) self._artifact_caches = artifact_caches self._backfill = backfill
def __init__(self, log, artifact_root, cache_root, compress=True, copy_fn=None): """ cache_root: The locally cached files are stored under this directory. copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that will copy cached files into the desired destination. If unspecified, a simple file copy is used. """ ArtifactCache.__init__(self, log, artifact_root) self._cache_root = os.path.expanduser(cache_root) self._compress = compress def copy(src, rel_dst): dst = os.path.join(self.artifact_root, rel_dst) safe_mkdir_for(dst) shutil.copy(src, dst) self._copy_fn = copy_fn or copy safe_mkdir(self._cache_root)
def __init__(self, log, artifact_root, url_base, compress=True): """ url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any path under this base. compress: Whether to compress the artifacts before storing them. """ ArtifactCache.__init__(self, log, artifact_root) parsed_url = urlparse.urlparse(url_base) if parsed_url.scheme == 'http': self._ssl = False elif parsed_url.scheme == 'https': self._ssl = True else: raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS') self._timeout_secs = 4.0 self._netloc = parsed_url.netloc self._path_prefix = parsed_url.path.rstrip('/') self.compress = compress
def __init__(self, read_artifact_cache, write_artifact_cache): """Either cache can be None, in which case we don't read from/write to it.""" artifact_roots = [] def get_root_and_log(cache): if cache is not None: artifact_roots.append(cache.artifact_root) get_root_and_log(read_artifact_cache) get_root_and_log(write_artifact_cache) if len(artifact_roots) == 0: # Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it. artifact_root = None else: artifact_root = artifact_roots[0] if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root: raise ValueError('Read and write artifact caches must have the same artifact root.') ArtifactCache.__init__(self, artifact_root) self._read_artifact_cache = read_artifact_cache self._write_artifact_cache = write_artifact_cache
def __init__(self, log, artifact_root, url_base, compress=True): """ url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any path under this base. compress: Whether to compress the artifacts before storing them. """ ArtifactCache.__init__(self, log, artifact_root) parsed_url = urlparse.urlparse(url_base) if parsed_url.scheme == 'http': self._ssl = False elif parsed_url.scheme == 'https': self._ssl = True else: raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS') self._timeout_secs = 4.0 self._netloc = parsed_url.netloc self._path_prefix = parsed_url.path.rstrip(b'/') self.compress = compress # To enable connection reuse, all requests must be created from same session. # TODO: Re-evaluate session's life-cycle if/when a longer-lived pants process exists. self._session = requests.Session()
def __init__(self, log, artifact_root, url_base, compress=True): """ url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any path under this base. compress: Whether to compress the artifacts before storing them. """ ArtifactCache.__init__(self, log, artifact_root) parsed_url = urlparse.urlparse(url_base) if parsed_url.scheme == 'http': self._ssl = False elif parsed_url.scheme == 'https': self._ssl = True else: raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS') self._timeout_secs = 4.0 self._netloc = parsed_url.netloc self._path_prefix = parsed_url.path.rstrip('/') self.compress = compress # Reduce the somewhat verbose logging of requests. # TODO do this in a central place logging.getLogger('requests').setLevel(logging.WARNING)
def __init__(self, log, artifact_root, url_base, compress=True): """ url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any path under this base. compress: Whether to compress the artifacts before storing them. """ ArtifactCache.__init__(self, log, artifact_root) parsed_url = urlparse.urlparse(url_base) if parsed_url.scheme == 'http': self._ssl = False elif parsed_url.scheme == 'https': self._ssl = True else: raise ValueError( 'RESTfulArtifactCache only supports HTTP and HTTPS') self._timeout_secs = 4.0 self._netloc = parsed_url.netloc self._path_prefix = parsed_url.path.rstrip('/') self.compress = compress # Reduce the somewhat verbose logging of requests. # TODO do this in a central place logging.getLogger('requests').setLevel(logging.WARNING)