Пример #1
0
    def __init__(self, read_artifact_cache, write_artifact_cache):
        """Either cache can be None, in which case we don't read from/write to it."""
        artifact_roots = []
        logs = []

        def get_root_and_log(cache):
            if cache is not None:
                artifact_roots.append(cache.artifact_root)
                logs.append(cache.log)

        get_root_and_log(read_artifact_cache)
        get_root_and_log(write_artifact_cache)
        if len(artifact_roots) == 0:
            # Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it.
            artifact_root = None
            log = None
        else:
            artifact_root = artifact_roots[0]
            log = logs[0]
            if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root:
                raise ValueError(
                    'Read and write artifact caches must have the same artifact root.'
                )
        ArtifactCache.__init__(self, log, artifact_root)
        self._read_artifact_cache = read_artifact_cache
        self._write_artifact_cache = write_artifact_cache
 def __init__(self, cache, pre_write_func=None, post_read_func=None):
   """
   cache: The underlying artifact cache.
   """
   ArtifactCache.__init__(self, cache.log, cache.artifact_root, cache.read_only)
   self._cache = cache
   self._pre_write_func = pre_write_func
   self._post_read_func = post_read_func
Пример #3
0
 def __init__(self, artifact_caches):
   if not artifact_caches:
     raise ValueError('Must provide at least one underlying artifact cache')
   log = artifact_caches[0].log
   artifact_root = artifact_caches[0].artifact_root
   if any(x.artifact_root != artifact_root for x in artifact_caches):
     raise ValueError('Combined artifact caches must all have the same artifact root.')
   ArtifactCache.__init__(self, log, artifact_root)
   self._artifact_caches = artifact_caches
Пример #4
0
 def __init__(self, cache, pre_write_func=None, post_read_func=None):
     """
 cache: The underlying artifact cache.
 """
     ArtifactCache.__init__(self, cache.log, cache.artifact_root,
                            cache.read_only)
     self._cache = cache
     self._pre_write_func = pre_write_func
     self._post_read_func = post_read_func
Пример #5
0
 def __init__(self, log, artifact_root, cache_root, copy_fn=None):
     """
 cache_root: The locally cached files are stored under this directory.
 copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
     will copy cached files into the desired destination. If unspecified, a simple file copy is used.
 """
     ArtifactCache.__init__(self, log, artifact_root)
     self._cache_root = cache_root
     self._copy_fn = copy_fn or (lambda src, rel_dst: shutil.copy(
         src, os.path.join(self.artifact_root, rel_dst)))
     safe_mkdir(self._cache_root)
 def __init__(self, log, artifact_root, cache_root, copy_fn=None):
   """
   cache_root: The locally cached files are stored under this directory.
   copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
       will copy cached files into the desired destination. If unspecified, a simple file copy is used.
   """
   ArtifactCache.__init__(self, log, artifact_root)
   self._cache_root = cache_root
   self._copy_fn = copy_fn or (
     lambda src, rel_dst: shutil.copy(src, os.path.join(self.artifact_root, rel_dst)))
   safe_mkdir(self._cache_root)
Пример #7
0
  def __init__(self, artifact_caches, backfill=True):
    """We delegate to artifact_caches, a list of ArtifactCache instances, in order.

    If backfill is true then we populate earlier caches that were missing an artifact,
    if that artifact was found in a later cache. This is useful for priming a local cache
    from a remote one.
    """
    if not artifact_caches:
      raise ValueError('Must provide at least one underlying artifact cache')
    log = artifact_caches[0].log
    artifact_root = artifact_caches[0].artifact_root
    if any(x.artifact_root != artifact_root for x in artifact_caches):
      raise ValueError('Combined artifact caches must all have the same artifact root.')
    ArtifactCache.__init__(self, log, artifact_root)
    self._artifact_caches = artifact_caches
    self._backfill = backfill
Пример #8
0
  def __init__(self, artifact_caches, backfill=True):
    """We delegate to artifact_caches, a list of ArtifactCache instances, in order.

    If backfill is true then we populate earlier caches that were missing an artifact,
    if that artifact was found in a later cache. This is useful for priming a local cache
    from a remote one.
    """
    if not artifact_caches:
      raise ValueError('Must provide at least one underlying artifact cache')
    log = artifact_caches[0].log
    artifact_root = artifact_caches[0].artifact_root
    if any(x.artifact_root != artifact_root for x in artifact_caches):
      raise ValueError('Combined artifact caches must all have the same artifact root.')
    read_only = all(x.read_only for x in artifact_caches)
    ArtifactCache.__init__(self, log, artifact_root, read_only)
    self._artifact_caches = artifact_caches
    self._backfill = backfill
Пример #9
0
  def __init__(self, log, artifact_root, cache_root, compress=True, copy_fn=None, read_only=False):
    """
    cache_root: The locally cached files are stored under this directory.
    copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
        will copy cached files into the desired destination. If unspecified, a simple file copy is used.
    """
    ArtifactCache.__init__(self, log, artifact_root, read_only)
    self._cache_root = os.path.expanduser(cache_root)
    self._compress = compress

    def copy(src, rel_dst):
      dst = os.path.join(self.artifact_root, rel_dst)
      safe_mkdir_for(dst)
      shutil.copy(src, dst)

    self._copy_fn = copy_fn or copy
    safe_mkdir(self._cache_root)
Пример #10
0
 def __init__(self, log, artifact_root, url_base, compress=True, read_only=False):
   """
   url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
             path under this base.
   compress: Whether to compress the artifacts before storing them.
   """
   ArtifactCache.__init__(self, log, artifact_root, read_only)
   parsed_url = urlparse.urlparse(url_base)
   if parsed_url.scheme == 'http':
     self._ssl = False
   elif parsed_url.scheme == 'https':
     self._ssl = True
   else:
     raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS')
   self._timeout_secs = 4.0
   self._netloc = parsed_url.netloc
   self._path_prefix = parsed_url.path.rstrip('/')
   self.compress = compress
Пример #11
0
 def __init__(self, log, artifact_root, url_base, compress=True):
   """
   url_base: The prefix for urls on some RESTful service. We must be able to PUT and GET to any
             path under this base.
   compress: Whether to compress the artifacts before storing them.
   """
   ArtifactCache.__init__(self, log, artifact_root)
   parsed_url = urlparse.urlparse(url_base)
   if parsed_url.scheme == 'http':
     self._ssl = False
   elif parsed_url.scheme == 'https':
     self._ssl = True
   else:
     raise ValueError('RESTfulArtifactCache only supports HTTP and HTTPS')
   self._timeout_secs = 4.0
   self._netloc = parsed_url.netloc
   self._path_prefix = parsed_url.path.rstrip('/')
   self.compress = compress
 def __init__(self, read_artifact_cache, write_artifact_cache):
   """Either cache can be None, in which case we don't read from/write to it."""
   artifact_roots = []
   logs = []
   def get_root_and_log(cache):
     if cache is not None:
       artifact_roots.append(cache.artifact_root)
       logs.append(cache.log)
   get_root_and_log(read_artifact_cache)
   get_root_and_log(write_artifact_cache)
   if len(artifact_roots) == 0:
     # Parent will never be accessed, so this is OK. In fact, it's a good way to ensure it.
     artifact_root = None
     log = None
   else:
     artifact_root = artifact_roots[0]
     log = logs[0]
     if len(artifact_roots) > 1 and artifact_roots[1] != artifact_root:
       raise ValueError('Read and write artifact caches must have the same artifact root.')
   ArtifactCache.__init__(self, log, artifact_root)
   self._read_artifact_cache = read_artifact_cache
   self._write_artifact_cache = write_artifact_cache
Пример #13
0
    def __init__(self,
                 log,
                 artifact_root,
                 cache_root,
                 compress=True,
                 copy_fn=None,
                 read_only=False):
        """
    cache_root: The locally cached files are stored under this directory.
    copy_fn: An optional function with the signature copy_fn(absolute_src_path, relative_dst_path) that
        will copy cached files into the desired destination. If unspecified, a simple file copy is used.
    """
        ArtifactCache.__init__(self, log, artifact_root, read_only)
        self._cache_root = os.path.expanduser(cache_root)
        self._compress = compress

        def copy(src, rel_dst):
            dst = os.path.join(self.artifact_root, rel_dst)
            safe_mkdir_for(dst)
            shutil.copy(src, dst)

        self._copy_fn = copy_fn or copy
        safe_mkdir(self._cache_root)