def request(self, name):
"""Returns an absolute path to the directory of the named cache.
Creates a cache directory if it does not exist yet.
Requires NamedCache to be open.
"""
self._lock.assert_locked()
assert isinstance(name, basestring), name
path = self._lru.get(name)
create_named_link = False
if path is None:
path = self._allocate_dir()
create_named_link = True
abs_path = os.path.join(self.root_dir, path)
file_path.ensure_tree(abs_path)
self._lru.add(name, path)
if create_named_link:
# Create symlink <root_dir>/<named>/<name> -> <root_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
fs.symlink(abs_path, named_path)
return abs_path
def test_cleanup_disk(self):
# Inject an item without a state.json, one is lost. Both will be deleted on
# cleanup.
self._free_disk = 1003
cache = self.get_cache(_get_policies(min_free_space=1000))
h_foo = self._algo('foo').hexdigest()
self.assertEqual([], sorted(cache._lru._items.iteritems()))
cache.write(h_foo, ['foo'])
self.assertEqual([], cache.trim())
self.assertEqual([h_foo],
[i[0] for i in cache._lru._items.iteritems()])
h_a = self._algo('a').hexdigest()
local_caching.file_write(os.path.join(cache.cache_dir, h_a), 'a')
# file_path.remove() explicitly handle the +R bit on Windows.
file_path.remove(os.path.join(cache.cache_dir, h_foo))
# Still hasn't realized that the file is missing.
self.assertEqual([h_foo],
[i[0] for i in cache._lru._items.iteritems()])
self.assertEqual(sorted([h_a, cache.STATE_FILE]),
sorted(fs.listdir(cache.cache_dir)))
cache.cleanup()
self.assertEqual([cache.STATE_FILE], fs.listdir(cache.cache_dir))
def uninstall(self, path, name):
"""Moves the cache directory back. Opposite to install().
NamedCache must be open. path must be absolute and unicode.
Raises NamedCacheError if cannot uninstall the cache.
"""
logging.info('Uninstalling named cache %r from %r', name, path)
with self._lock:
try:
if not os.path.isdir(path):
logging.warning(
'Directory %r does not exist anymore. Cache lost.',
path)
return
rel_cache = self._lru.get(name)
if rel_cache:
# Do not crash because cache already exists.
logging.warning('overwriting an existing named cache %r',
name)
create_named_link = False
else:
rel_cache = self._allocate_dir()
create_named_link = True
# Move the dir and create an entry for the named cache.
abs_cache = os.path.join(self.cache_dir, rel_cache)
logging.info('Moving %r to %r', path, abs_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(path, abs_cache)
self._lru.add(name, rel_cache)
if create_named_link:
# Create symlink <cache_dir>/<named>/<name> -> <cache_dir>/<short
# name> for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
try:
fs.symlink(abs_cache, named_path)
logging.info('Created symlink %r to %r', named_path,
abs_cache)
except OSError:
# Ignore on Windows. It happens when running as a normal user or
# when UAC is enabled and the user is a filtered administrator
# account.
if sys.platform != 'win32':
raise
except (IOError, OSError) as ex:
raise NamedCacheError(
'cannot uninstall cache named %r at %r: %s' %
(name, path, ex))
def uninstall(self, path, name):
"""Moves the cache directory back. Opposite to install().
NamedCache must be open. path must be absolute and unicode.
Raises Error if cannot uninstall the cache.
"""
logging.info('Uninstalling named cache %r from %r', name, path)
try:
_check_abs(path)
if not os.path.isdir(path):
logging.warning(
'Directory %r does not exist anymore. Cache lost.', path)
return
rel_cache = self._lru.get(name)
if rel_cache:
# Do not crash because cache already exists.
logging.warning('overwriting an existing named cache %r', name)
create_named_link = False
else:
rel_cache = self._allocate_dir()
create_named_link = True
# Move the dir and create an entry for the named cache.
abs_cache = os.path.join(self.root_dir, rel_cache)
logging.info('Moving %r to %r', path, abs_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(path, abs_cache)
self._lru.add(name, rel_cache)
if create_named_link:
# Create symlink <root_dir>/<named>/<name> -> <root_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
fs.symlink(abs_cache, named_path)
logging.info('Created symlink %r to %r', named_path, abs_cache)
except (OSError, Error) as ex:
raise Error(
'cannot uninstall cache named %r at %r: %s' % (
name, path, ex))
def test_some_file_brutally_deleted(self):
h_a = self._algo('a').hexdigest()
self._free_disk = 1100
cache = self.get_cache(_get_policies())
cache.write(h_a, 'a')
self.assertTrue(cache.touch(h_a, local_caching.UNKNOWN_FILE_SIZE))
self.assertTrue(cache.touch(h_a, 1))
self.assertEqual([], cache.trim())
# file_path.remove() explicitly handle the +R bit on Windows.
file_path.remove(os.path.join(cache.cache_dir, h_a))
cache = self.get_cache(_get_policies())
# 'Ghost' entry loaded with state.json is still there.
self.assertEqual([h_a], list(cache))
# 'touch' detects the file is missing by returning False.
self.assertFalse(cache.touch(h_a, local_caching.UNKNOWN_FILE_SIZE))
self.assertFalse(cache.touch(h_a, 1))
# 'touch' evicted the entry.
self.assertEqual([], list(cache))
def cleanup_bot_directory(botobj):
"""Delete anything not expected in the swarming bot directory.
This helps with stale work directory or any unexpected junk that could cause
this bot to self-quarantine. Do only this when running from the zip.
"""
if not is_base_dir_ok(botobj):
# That's an important one-off check as cleaning the $HOME directory has
# really bad effects on normal host.
logging.error(
'Not cleaning root directory because of bad base directory')
return
for i in os.listdir(botobj.base_dir):
if any(fnmatch.fnmatch(i, w) for w in PASSLIST):
continue
try:
p = unicode(os.path.join(botobj.base_dir, i))
if os.path.isdir(p):
file_path.rmtree(p)
else:
file_path.remove(p)
except (IOError, OSError) as e:
botobj.post_error('Failed to remove %s from bot\'s directory: %s' %
(i, e))
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'versions')),
isolateserver.CachePolicies(0, 0, 300),
hashlib.sha1,
trim=True)
with version_cache:
version_cache.cleanup()
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except isolateserver.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'clients')),
isolateserver.CachePolicies(0, 0, 5),
hashlib.sha1,
trim=True)
with instance_cache:
instance_cache.cleanup()
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name,
instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url,
timeoutfn)
# A single host can run multiple swarming bots, but ATM they do not share
# same root bot directory. Thus, it is safe to use the same name for the
# binary.
cipd_bin_dir = unicode(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
file_path.remove(binary_path)
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'versions')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=300,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except local_caching.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
version_cache.trim()
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'clients')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=10,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name, instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn)
# A single host can run multiple swarming bots, but they cannot share same
# root bot directory. Thus, it is safe to use the same name for the binary.
cipd_bin_dir = six.text_type(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
# TODO(maruel): Do not unconditionally remove the binary.
try:
file_path.remove(binary_path)
except WindowsError: # pylint: disable=undefined-variable
# See whether cipd.exe is running for crbug.com/1028781
ret = subprocess42.call(['tasklist.exe'])
if ret:
logging.error('tasklist returns non-zero: %d', ret)
raise
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0o511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
instance_cache.trim()
def uninstall(self, path, name):
"""Moves the cache directory back. Opposite to install().
path must be absolute and unicode.
Raises NamedCacheError if cannot uninstall the cache.
"""
logging.info('Uninstalling named cache %r from %r', name, path)
with self._lock:
try:
if not os.path.isdir(path):
logging.warning(
'Directory %r does not exist anymore. Cache lost.',
path)
return
if name in self._lru:
# This shouldn't happen but just remove the preexisting one and move
# on.
logging.warning('overwriting an existing named cache %r',
name)
self._remove(name)
rel_cache = self._allocate_dir()
# Move the dir and create an entry for the named cache.
abs_cache = os.path.join(self.cache_dir, rel_cache)
logging.info('Moving %r to %r', path, abs_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(path, abs_cache)
# That succeeded, calculate its new size.
size = _get_recursive_size(abs_cache)
if not size:
# Do not save empty named cache.
return
self._lru.add(name, (rel_cache, size))
self._added.append(size)
# Create symlink <cache_dir>/<named>/<name> -> <cache_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
try:
fs.symlink(abs_cache, named_path)
logging.info('Created symlink %r to %r', named_path,
abs_cache)
except OSError:
# Ignore on Windows. It happens when running as a normal user or when
# UAC is enabled and the user is a filtered administrator account.
if sys.platform != 'win32':
raise
except (IOError, OSError) as ex:
raise NamedCacheError(
'cannot uninstall cache named %r at %r: %s' %
(name, path, ex))
finally:
self._save()
def uninstall(self, src, name):
"""Moves the cache directory back into the named cache hive for an eventual
reuse.
The opposite of install().
src must be absolute and unicode. Its content is moved back into the local
named caches cache.
Returns the named cache size in bytes.
Raises NamedCacheError if cannot uninstall the cache.
"""
logging.info('NamedCache.uninstall(%r, %r)', src, name)
with self._lock:
try:
if not fs.isdir(src):
logging.warning(
'NamedCache: Directory %r does not exist anymore. Cache lost.',
src)
return
if name in self._lru:
# This shouldn't happen but just remove the preexisting one and move
# on.
logging.error('- overwriting existing cache!')
self._remove(name)
# Calculate the size of the named cache to keep. It's important because
# if size is zero (it's empty), we do not want to add it back to the
# named caches cache.
size = _get_recursive_size(src)
logging.info('- Size is %d', size)
if not size:
# Do not save empty named cache.
return size
# Move the dir and create an entry for the named cache.
rel_cache = self._allocate_dir()
abs_cache = os.path.join(self.cache_dir, rel_cache)
logging.info('- Moving to %r', rel_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(src, abs_cache)
self._lru.add(name, (rel_cache, size))
self._added.append(size)
# Create symlink <cache_dir>/<named>/<name> -> <cache_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if fs.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
try:
fs.symlink(os.path.join(u'..', rel_cache), named_path)
logging.info('NamedCache: Created symlink %r to %r',
named_path, abs_cache)
except OSError:
# Ignore on Windows. It happens when running as a normal user or when
# UAC is enabled and the user is a filtered administrator account.
if sys.platform != 'win32':
raise
return size
except (IOError, OSError) as ex:
# Raise using the original traceback.
exc = NamedCacheError(
'cannot uninstall cache named %r at %r: %s' %
(name, src, ex))
six.reraise(exc, None, sys.exc_info()[2])
finally:
# Call save() at every uninstall. The assumptions are:
# - The total the number of named caches is low, so the state.json file
# is small, so the time it takes to write it to disk is short.
# - The number of mapped named caches per task is low, so the number of
# times save() is called on tear-down isn't high enough to be
# significant.
# - uninstall() sometimes throws due to file locking on Windows or
# access rights on Linux. We want to keep as many as possible.
self._save()
