def process_named_cache_options(parser, options, time_fn=None):
"""Validates named cache options and returns a CacheManager."""
if options.named_caches and not options.named_cache_root:
parser.error('--named-cache is specified, but --named-cache-root is empty')
for name, path in options.named_caches:
if not CACHE_NAME_RE.match(name):
parser.error(
'cache name %r does not match %r' % (name, CACHE_NAME_RE.pattern))
if not path:
parser.error('cache path cannot be empty')
if options.named_cache_root:
# Make these configurable later if there is use case but for now it's fairly
# safe values.
# In practice, a fair chunk of bots are already recycled on a daily schedule
# so this code doesn't have any effect to them, unless they are preloaded
# with a really old cache.
policies = local_caching.CachePolicies(
# 1TiB.
max_cache_size=1024*1024*1024*1024,
min_free_space=options.min_free_space,
max_items=50,
max_age_secs=MAX_AGE_SECS)
root_dir = unicode(os.path.abspath(options.named_cache_root))
return local_caching.NamedCache(root_dir, policies, time_fn=time_fn)
return None
def setUp(self):
super(NamedCacheTest, self).setUp()
self.policies = local_caching.CachePolicies(
max_cache_size=1024 * 1024 * 1024,
min_free_space=1024,
max_items=50,
max_age_secs=21 * 24 * 60 * 60)
self.cache_dir = os.path.join(self.tempdir, 'cache')
def _get_policies(max_cache_size=0,
min_free_space=0,
max_items=0,
max_age_secs=0):
"""Returns a CachePolicies with only the policy we want to enforce."""
return local_caching.CachePolicies(max_cache_size=max_cache_size,
min_free_space=min_free_space,
max_items=max_items,
max_age_secs=max_age_secs)
def test_policies_trim_old(self):
# Add two items, one 3 weeks and one minute old, one recent, make sure the
# old one is trimmed.
self._policies = local_caching.CachePolicies(max_cache_size=1000,
min_free_space=0,
max_items=1000,
max_age_secs=21 * 24 *
60 * 60)
now = 100
c = self.get_cache(time_fn=lambda: now)
# Test the very limit of 3 weeks:
c.write(hashlib.sha1('old').hexdigest(), 'old')
now += 1
c.write(hashlib.sha1('recent').hexdigest(), 'recent')
now += 21 * 24 * 60 * 60
c.trim()
self.assertEqual(set([hashlib.sha1('recent').hexdigest()]),
c.cached_set())
def setUp(self):
super(DiskContentAddressedCacheTest, self).setUp()
# If this fails on Windows, please rerun this tests as an elevated user with
# administrator access right.
self.assertEqual(True, file_path.enable_symlink())
self._algo = hashlib.sha256
self._free_disk = 1000
# Max: 100 bytes, 2 items
# Min free disk: 1000 bytes.
self._policies = local_caching.CachePolicies(max_cache_size=100,
min_free_space=1000,
max_items=2,
max_age_secs=0)
def get_free_space(p):
self.assertEqual(p, os.path.join(self.tempdir, 'cache'))
return self._free_disk
self.mock(file_path, 'get_free_space', get_free_space)
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'versions')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=300,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except local_caching.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
version_cache.trim()
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'clients')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=10,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name, instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn)
# A single host can run multiple swarming bots, but they cannot share same
# root bot directory. Thus, it is safe to use the same name for the binary.
cipd_bin_dir = six.text_type(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
# TODO(maruel): Do not unconditionally remove the binary.
try:
file_path.remove(binary_path)
except WindowsError: # pylint: disable=undefined-variable
# See whether cipd.exe is running for crbug.com/1028781
ret = subprocess42.call(['tasklist.exe'])
if ret:
logging.error('tasklist returns non-zero: %d', ret)
raise
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0o511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
instance_cache.trim()
def test_run_command_caches(self):
# This test puts a file into a named cache, remove it, runs a test that
# updates the named cache, remaps it and asserts the content was updated.
#
# Directories:
# <root_dir>/
# <root_dir>/c - <cache_dir> named cache root
# <root_dir>/dest - <dest_dir> used for manual cache update
# <root_dir>/w - <self.work_dir> used by the task.
cache_dir = os.path.join(self.root_dir, u'c')
dest_dir = os.path.join(self.root_dir, u'dest')
policies = local_caching.CachePolicies(0, 0, 0, 0)
# Inject file 'bar' in the named cache 'foo'.
cache = local_caching.NamedCache(cache_dir, policies)
cache.install(dest_dir, 'foo')
with open(os.path.join(dest_dir, 'bar'), 'wb') as f:
f.write('thecache')
cache.uninstall(dest_dir, 'foo')
self.assertFalse(os.path.exists(dest_dir))
self._expect_files([u'c/*/bar', u'c/state.json'])
# Maps the cache 'foo' as 'cache_foo'. This runs inside self.work_dir.
# This runs the command for real.
script = (
'import os\n'
'print "hi"\n'
'with open("cache_foo/bar", "rb") as f:\n'
' cached = f.read()\n'
'with open("../../result", "wb") as f:\n'
' f.write(cached)\n'
'with open("cache_foo/bar", "wb") as f:\n'
' f.write("updated_cache")\n')
task_details = get_task_details(
script, caches=[{'name': 'foo', 'path': 'cache_foo', 'hint': '100'}])
expected = {
u'exit_code': 0,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
self.assertEqual(expected, self._run_command(task_details))
self._expect_files(
[
u'c/*/bar', u'c/state.json', u'result', u'w/run_isolated_args.json',
])
# Ensure the 'result' file written my the task contained foo/bar.
with open(os.path.join(self.root_dir, 'result'), 'rb') as f:
self.assertEqual('thecache', f.read())
os.remove(os.path.join(self.root_dir, 'result'))
cache = local_caching.NamedCache(cache_dir, policies)
self.assertFalse(os.path.exists(dest_dir))
self._expect_files(
[u'c/*/bar', u'c/state.json', u'w/run_isolated_args.json'])
cache.install(dest_dir, 'foo')
self._expect_files(
[u'dest/bar', u'c/state.json', u'w/run_isolated_args.json'])
with open(os.path.join(dest_dir, 'bar'), 'rb') as f:
self.assertEqual('updated_cache', f.read())
cache.uninstall(dest_dir, 'foo')
self.assertFalse(os.path.exists(dest_dir))
# Now look at the updates sent by the bot as seen by the server.
self.expectTask()
def test_policies_active_trimming(self):
# Start with a larger cache, add many object.
# Reload the cache with smaller policies, the cache should be trimmed on
# load.
h_a = self.to_hash('a')[0]
h_b = self.to_hash('b')[0]
h_c = self.to_hash('c')[0]
h_large, large = self.to_hash('b' * 99)
def assertItems(expected):
actual = [(digest, size)
for digest, (size, _) in cache._lru._items.iteritems()]
self.assertEqual(expected, actual)
# Max policies is 100 bytes, 2 items, 1000 bytes free space.
self._free_disk = 1101
with self.get_cache() as cache:
cache.write(h_a, 'a')
cache.write(h_large, large)
# Cache (size and # items) is not enforced while adding items. The
# rationale is that a task may request more data than the size of the
# cache policies. As long as there is free space, this is fine.
cache.write(h_b, 'b')
assertItems([(h_a, 1), (h_large, len(large)), (h_b, 1)])
self.assertEqual(h_a, cache._protected)
self.assertEqual(1000, cache._free_disk)
self.assertEqual(0, cache.initial_number_items)
self.assertEqual(0, cache.initial_size)
# Free disk is enforced, because otherwise we assume the task wouldn't
# be able to start. In this case, it throws an exception since all items
# are protected. The item is added since it's detected after the fact.
with self.assertRaises(local_caching.NoMoreSpace):
cache.write(h_c, 'c')
# At this point, after the implicit trim in __exit__(), h_a and h_large were
# evicted.
self.assertEqual(sorted([h_b, h_c, u'state.json']),
sorted(os.listdir(cache.cache_dir)))
# Allow 3 items and 101 bytes so h_large is kept.
self._policies = local_caching.CachePolicies(max_cache_size=101,
min_free_space=1000,
max_items=3,
max_age_secs=0)
with self.get_cache() as cache:
cache.write(h_large, large)
self.assertEqual(2, cache.initial_number_items)
self.assertEqual(2, cache.initial_size)
self.assertEqual(sorted([h_b, h_c, h_large, u'state.json']),
sorted(os.listdir(cache.cache_dir)))
# Assert that trimming is done in constructor too.
self._policies = local_caching.CachePolicies(max_cache_size=100,
min_free_space=1000,
max_items=2,
max_age_secs=0)
with self.get_cache() as cache:
assertItems([(h_c, 1), (h_large, len(large))])
self.assertEqual(None, cache._protected)
self.assertEqual(1101, cache._free_disk)
self.assertEqual(2, cache.initial_number_items)
self.assertEqual(100, cache.initial_size)