def test_policies_active_trimming(self):
# Start with a larger cache, add many object.
# Reload the cache with smaller policies, the cache should be trimmed on
# load.
h_a = self.to_hash('a')[0]
h_b = self.to_hash('b')[0]
h_c = self.to_hash('c')[0]
h_large, large = self.to_hash('b' * 99)
def assertItems(expected):
actual = [
(digest, size) for digest, (size, _) in cache._lru._items.iteritems()]
self.assertEqual(expected, actual)
# Max policies is 100 bytes, 2 items, 1000 bytes free space.
self._free_disk = 1101
with self.get_cache() as cache:
cache.write(h_a, 'a')
cache.write(h_large, large)
# Cache (size and # items) is not enforced while adding items. The
# rationale is that a task may request more data than the size of the
# cache policies. As long as there is free space, this is fine.
cache.write(h_b, 'b')
assertItems([(h_a, 1), (h_large, len(large)), (h_b, 1)])
self.assertEqual(h_a, cache._protected)
self.assertEqual(1000, cache._free_disk)
self.assertEqual(0, cache.initial_number_items)
self.assertEqual(0, cache.initial_size)
# Free disk is enforced, because otherwise we assume the task wouldn't
# be able to start. In this case, it throws an exception since all items
# are protected. The item is added since it's detected after the fact.
with self.assertRaises(isolateserver.Error):
cache.write(h_c, 'c')
# At this point, after the implicit trim in __exit__(), h_a and h_large were
# evicted.
self.assertEqual(
sorted([h_b, h_c, u'state.json']), sorted(os.listdir(self.tempdir)))
# Allow 3 items and 101 bytes so h_large is kept.
self._policies = isolateserver.CachePolicies(101, 1000, 3)
with self.get_cache() as cache:
cache.write(h_large, large)
self.assertEqual(2, cache.initial_number_items)
self.assertEqual(2, cache.initial_size)
self.assertEqual(
sorted([h_b, h_c, h_large, u'state.json']),
sorted(os.listdir(self.tempdir)))
# Assert that trimming is done in constructor too.
self._policies = isolateserver.CachePolicies(100, 1000, 2)
with self.get_cache() as cache:
assertItems([(h_c, 1), (h_large, len(large))])
self.assertEqual(None, cache._protected)
self.assertEqual(1101, cache._free_disk)
self.assertEqual(2, cache.initial_number_items)
self.assertEqual(100, cache.initial_size)
def setUp(self):
super(DiskCacheTest, self).setUp()
self._algo = isolated_format.get_hash_algo('default-gzip')
self._free_disk = 1000
# Max: 100 bytes, 2 items
# Min free disk: 1000 bytes.
self._policies = isolateserver.CachePolicies(100, 1000, 2)
def get_free_space(p):
self.assertEqual(p, self.tempdir)
return self._free_disk
self.mock(file_path, 'get_free_space', get_free_space)
def setUp(self):
super(DiskCacheTest, self).setUp()
# If this fails on Windows, please rerun this tests as an elevated user with
# administrator access right.
self.assertEqual(True, file_path.enable_symlink())
self._algo = isolated_format.get_hash_algo('default-gzip')
self._free_disk = 1000
# Max: 100 bytes, 2 items
# Min free disk: 1000 bytes.
self._policies = isolateserver.CachePolicies(100, 1000, 2)
def get_free_space(p):
self.assertEqual(p, self.tempdir)
return self._free_disk
self.mock(file_path, 'get_free_space', get_free_space)
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'versions')),
isolateserver.CachePolicies(0, 0, 300),
hashlib.sha1,
trim=True)
with version_cache:
version_cache.cleanup()
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except isolateserver.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'clients')),
isolateserver.CachePolicies(0, 0, 5),
hashlib.sha1,
trim=True)
with instance_cache:
instance_cache.cleanup()
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name,
instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url,
timeoutfn)
# A single host can run multiple swarming bots, but ATM they do not share
# same root bot directory. Thus, it is safe to use the same name for the
# binary.
cipd_bin_dir = unicode(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
file_path.remove(binary_path)
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)