def test_clean_caches_disk(self): # Create an isolated cache and a named cache each with 2 items. Ensure that # one item from each is removed. now = self._now self._free_disk = 100000 # Setup caches. policies = _get_policies(min_free_space=1000) named_cache = local_caching.NamedCache( tempfile.mkdtemp(dir=self.tempdir, prefix='nc'), policies) short_names = self._prepare_named_cache(named_cache) isolated_cache = local_caching.DiskContentAddressedCache( tempfile.mkdtemp(dir=self.tempdir, prefix='ic'), policies, trim=False) self._prepare_isolated_cache(isolated_cache) self.assertEqual(now, self._now) # Request trimming. self._free_disk = 950 trimmed = local_caching.trim_caches( [isolated_cache, named_cache], self.tempdir, min_free_space=policies.min_free_space, max_age_secs=policies.max_age_secs) # Enough to free 50 bytes. The following sums to 56. expected = [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7] self.assertEqual(expected, trimmed) # Cache verification. self._verify_named_cache(named_cache, short_names, range(8, 11)) self._verify_isolated_cache(isolated_cache, range(8, 11))
def get_cache(self, **kwargs): cache_dir = os.path.join(self.tempdir, 'cache') return local_caching.DiskContentAddressedCache(cache_dir, self._policies, self._algo, trim=True, **kwargs)
def get_cache(self, policies): return local_caching.DiskContentAddressedCache(os.path.join( self.tempdir, 'cache'), policies, trim=True)
def get_client(service_url, package_template, version, cache_dir, timeout=None): """Returns a context manager that yields a CipdClient. A blocking call. Upon exit from the context manager, the client binary may be deleted (if the internal cache is full). Args: service_url (str): URL of the CIPD backend. package_template (str): package name template of the CIPD client. version (str): version of CIPD client package. cache_dir: directory to store instance cache, version cache and a hardlink to the client binary. timeout (int): if not None, timeout in seconds for this function. Yields: CipdClient. Raises: Error if CIPD client version cannot be resolved or client cannot be fetched. """ timeoutfn = tools.sliding_timeout(timeout) # Package names are always lower case. # TODO(maruel): Assert instead? package_name = package_template.lower().replace('${platform}', get_platform()) # Resolve version to instance id. # Is it an instance id already? They look like HEX SHA1. if isolated_format.is_valid_hash(version, hashlib.sha1): instance_id = version elif ':' in version: # it's an immutable tag, cache the resolved version # version_cache is {hash(package_name, tag) -> instance id} mapping. # It does not take a lot of disk space. version_cache = local_caching.DiskContentAddressedCache( six.text_type(os.path.join(cache_dir, 'versions')), local_caching.CachePolicies( # 1GiB. max_cache_size=1024 * 1024 * 1024, min_free_space=0, max_items=300, # 3 weeks. max_age_secs=21 * 24 * 60 * 60), trim=True) # Convert (package_name, version) to a string that may be used as a # filename in disk cache by hashing it. version_digest = hashlib.sha1('%s\n%s' % (package_name, version)).hexdigest() try: with version_cache.getfileobj(version_digest) as f: instance_id = f.read() except local_caching.CacheMiss: instance_id = resolve_version(service_url, package_name, version, timeout=timeoutfn()) version_cache.write(version_digest, instance_id) version_cache.trim() else: # it's a ref, hit the backend instance_id = resolve_version(service_url, package_name, version, timeout=timeoutfn()) # instance_cache is {instance_id -> client binary} mapping. # It is bounded by 5 client versions. instance_cache = local_caching.DiskContentAddressedCache( six.text_type(os.path.join(cache_dir, 'clients')), local_caching.CachePolicies( # 1GiB. max_cache_size=1024 * 1024 * 1024, min_free_space=0, max_items=10, # 3 weeks. max_age_secs=21 * 24 * 60 * 60), trim=True) if instance_id not in instance_cache: logging.info('Fetching CIPD client %s:%s', package_name, instance_id) fetch_url = get_client_fetch_url(service_url, package_name, instance_id, timeout=timeoutfn()) _fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn) # A single host can run multiple swarming bots, but they cannot share same # root bot directory. Thus, it is safe to use the same name for the binary. cipd_bin_dir = six.text_type(os.path.join(cache_dir, 'bin')) binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX) if fs.isfile(binary_path): # TODO(maruel): Do not unconditionally remove the binary. try: file_path.remove(binary_path) except WindowsError: # pylint: disable=undefined-variable # See whether cipd.exe is running for crbug.com/1028781 ret = subprocess42.call(['tasklist.exe']) if ret: logging.error('tasklist returns non-zero: %d', ret) raise else: file_path.ensure_tree(cipd_bin_dir) with instance_cache.getfileobj(instance_id) as f: isolateserver.putfile(f, binary_path, 0o511) # -r-x--x--x _ensure_batfile(binary_path) yield CipdClient(binary_path, package_name=package_name, instance_id=instance_id, service_url=service_url) instance_cache.trim()
def get_cache(self, policies): return local_caching.DiskContentAddressedCache(self.cache_dir(), policies, trim=True)