def __init__(self, cache_dir, policies, hash_algo, trim, time_fn=None):
"""
Arguments:
cache_dir: directory where to place the cache.
policies: CachePolicies instance, cache retention policies.
algo: hashing algorithm used.
trim: if True to enforce |policies| right away.
It can be done later by calling trim() explicitly.
"""
# All protected methods (starting with '_') except _path should be called
# with self._lock held.
super(DiskContentAddressedCache, self).__init__(cache_dir)
self.policies = policies
self.hash_algo = hash_algo
self.state_file = os.path.join(cache_dir, self.STATE_FILE)
# Items in a LRU lookup dict(digest: size).
self._lru = lru.LRUDict()
# Current cached free disk space. It is updated by self._trim().
file_path.ensure_tree(self.cache_dir)
self._free_disk = file_path.get_free_space(self.cache_dir)
# The first item in the LRU cache that must not be evicted during this run
# since it was referenced. All items more recent that _protected in the LRU
# cache are also inherently protected. It could be a set() of all items
# referenced but this increases memory usage without a use case.
self._protected = None
# Cleanup operations done by self._load(), if any.
self._operations = []
with tools.Profiler('Setup'):
with self._lock:
self._load(trim, time_fn)
def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
"""Creates a isolated tree usable for test execution.
Returns the current working directory where the isolated command should be
started in.
"""
# Forcibly copy when the tree has to be read only. Otherwise the inode is
# modified, and this cause real problems because the user's source tree
# becomes read only. On the other hand, the cost of doing file copy is huge.
if read_only not in (0, None):
action = file_path.COPY
else:
action = file_path.HARDLINK_WITH_FALLBACK
recreate_tree(outdir=outdir,
indir=root_dir,
infiles=files,
action=action,
as_hash=False)
cwd = os.path.normpath(os.path.join(outdir, relative_cwd))
# cwd may not exist when no files are mapped from the directory containing the
# .isolate file. But the directory must exist to be the current working
# directory.
file_path.ensure_tree(cwd)
run_isolated.change_tree_read_only(outdir, read_only)
return cwd
def install(self, path, name):
"""Moves the directory for the specified named cache to |path|.
NamedCache must be open. path must be absolute, unicode and must not exist.
Raises Error if cannot install the cache.
"""
self._lock.assert_locked()
logging.info('Installing named cache %r to %r', name, path)
try:
_check_abs(path)
if os.path.isdir(path):
raise Error('installation directory %r already exists' % path)
rel_cache = self._lru.get(name)
if rel_cache:
abs_cache = os.path.join(self.root_dir, rel_cache)
if os.path.isdir(abs_cache):
logging.info('Moving %r to %r', abs_cache, path)
file_path.ensure_tree(os.path.dirname(path))
fs.rename(abs_cache, path)
self._remove(name)
return
logging.warning('directory for named cache %r does not exist',
name)
self._remove(name)
# The named cache does not exist, create an empty directory.
# When uninstalling, we will move it back to the cache and create an
# an entry.
file_path.ensure_tree(path)
except (OSError, Error) as ex:
raise Error('cannot install cache named %r at %r: %s' %
(name, path, ex))
def CMDremap(parser, args):
"""Creates a directory with all the dependencies mapped into it.
Useful to test manually why a test is failing. The target executable is not
run.
"""
add_isolate_options(parser)
add_outdir_options(parser)
add_skip_refresh_option(parser)
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
cwd = os.getcwd()
process_isolate_options(parser, options, cwd, require_isolated=False)
process_outdir_options(parser, options, cwd)
complete_state = load_complete_state(options, cwd, None,
options.skip_refresh)
file_path.ensure_tree(options.outdir)
print('Remapping into %s' % options.outdir)
if fs.listdir(options.outdir):
raise ExecutionError('Can\'t remap in a non-empty directory')
create_isolate_tree(options.outdir, complete_state.root_dir,
complete_state.saved_state.files,
complete_state.saved_state.relative_cwd,
complete_state.saved_state.read_only)
if complete_state.isolated_filepath:
complete_state.save_files()
return 0
def open(self, time_fn=None):
"""Opens NamedCaches for mutation operations, such as install.
Only one caller can open the cache manager at a time. If the same thread
calls this function after opening it earlier, the call will deadlock.
time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested.
Returns a context manager that must be closed as soon as possible.
"""
with self._lock:
state_path = os.path.join(self.root_dir, u'state.json')
assert self._lru is None, 'acquired lock, but self._lru is not None'
if os.path.isfile(state_path):
try:
self._lru = lru.LRUDict.load(state_path)
except ValueError:
logging.exception('failed to load named cache state file')
logging.warning('deleting named caches')
file_path.rmtree(self.root_dir)
self._lru = self._lru or lru.LRUDict()
if time_fn:
self._lru.time_fn = time_fn
try:
yield
finally:
file_path.ensure_tree(self.root_dir)
self._lru.save(state_path)
self._lru = None
def open(self, time_fn=None):
"""Opens NamedCaches for mutation operations, such as request or trim.
Only on caller can open the cache manager at a time. If the same thread
calls this function after opening it earlier, the call will deadlock.
time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested.
Returns a context manager that must be closed as soon as possible.
"""
state_path = os.path.join(self.root_dir, u'state.json')
with self._lock:
if os.path.isfile(state_path):
self._lru = lru.LRUDict.load(state_path)
else:
self._lru = lru.LRUDict()
if time_fn:
self._lru.time_fn = time_fn
try:
yield
finally:
file_path.ensure_tree(self.root_dir)
self._lru.save(state_path)
self._lru = None
def request(self, name):
"""Returns an absolute path to the directory of the named cache.
Creates a cache directory if it does not exist yet.
Requires NamedCache to be open.
"""
self._lock.assert_locked()
assert isinstance(name, basestring), name
path = self._lru.get(name)
create_named_link = False
if path is None:
path = self._allocate_dir()
create_named_link = True
abs_path = os.path.join(self.root_dir, path)
file_path.ensure_tree(abs_path)
self._lru.add(name, path)
if create_named_link:
# Create symlink <root_dir>/<named>/<name> -> <root_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
fs.symlink(abs_path, named_path)
return abs_path
def test_ensure_tree(self):
dir_foo = os.path.join(self.tempdir, 'foo')
file_path.ensure_tree(dir_foo, 0o777)
self.assertTrue(os.path.isdir(dir_foo))
# Do not raise OSError with errno.EEXIST
file_path.ensure_tree(dir_foo, 0o777)
def uninstall(self, path, name):
"""Moves the cache directory back. Opposite to install().
NamedCache must be open. path must be absolute and unicode.
Raises NamedCacheError if cannot uninstall the cache.
"""
logging.info('Uninstalling named cache %r from %r', name, path)
with self._lock:
try:
if not os.path.isdir(path):
logging.warning(
'Directory %r does not exist anymore. Cache lost.',
path)
return
rel_cache = self._lru.get(name)
if rel_cache:
# Do not crash because cache already exists.
logging.warning('overwriting an existing named cache %r',
name)
create_named_link = False
else:
rel_cache = self._allocate_dir()
create_named_link = True
# Move the dir and create an entry for the named cache.
abs_cache = os.path.join(self.cache_dir, rel_cache)
logging.info('Moving %r to %r', path, abs_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(path, abs_cache)
self._lru.add(name, rel_cache)
if create_named_link:
# Create symlink <cache_dir>/<named>/<name> -> <cache_dir>/<short
# name> for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
try:
fs.symlink(abs_cache, named_path)
logging.info('Created symlink %r to %r', named_path,
abs_cache)
except OSError:
# Ignore on Windows. It happens when running as a normal user or
# when UAC is enabled and the user is a filtered administrator
# account.
if sys.platform != 'win32':
raise
except (IOError, OSError) as ex:
raise NamedCacheError(
'cannot uninstall cache named %r at %r: %s' %
(name, path, ex))
def run_tha_test(data, result_json):
"""Runs an executable and records execution metadata.
If isolated_hash is specified, downloads the dependencies in the cache,
hardlinks them into a temporary directory and runs the command specified in
the .isolated.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
- data: TaskData instance.
- result_json: File path to dump result metadata into. If set, the process
exit code is always 0 unless an internal error occurred.
Returns:
Process exit code that should be used.
"""
if result_json:
# Write a json output file right away in case we get killed.
result = {
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': 'Was terminated before completion',
'outputs_ref': None,
'version': 5,
}
tools.write_json(result_json, result, dense=True)
# run_isolated exit code. Depends on if result_json is used or not.
result = map_and_run(data, True)
logging.info('Result:\n%s', tools.format_json(result, dense=True))
if result_json:
# We've found tests to delete 'work' when quitting, causing an exception
# here. Try to recreate the directory if necessary.
file_path.ensure_tree(os.path.dirname(result_json))
tools.write_json(result_json, result, dense=True)
# Only return 1 if there was an internal error.
return int(bool(result['internal_failure']))
# Marshall into old-style inline output.
if result['outputs_ref']:
data = {
'hash': result['outputs_ref']['isolated'],
'namespace': result['outputs_ref']['namespace'],
'storage': result['outputs_ref']['isolatedserver'],
}
sys.stdout.flush()
print(
'[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
tools.format_json(data, dense=True))
sys.stdout.flush()
return result['exit_code'] or int(bool(result['internal_failure']))
def install(self, dst, name):
"""Creates the directory |dst| and moves a previous named cache |name| if it
was in the local named caches cache.
dst must be absolute, unicode and must not exist.
Returns the reused named cache size in bytes, or 0 if none was present.
Raises NamedCacheError if cannot install the cache.
"""
logging.info('NamedCache.install(%r, %r)', dst, name)
with self._lock:
try:
if fs.isdir(dst):
raise NamedCacheError(
'installation directory %r already exists' % dst)
# Remove the named symlink if it exists.
link_name = self._get_named_path(name)
if fs.exists(link_name):
# Remove the symlink itself, not its destination.
fs.remove(link_name)
if name in self._lru:
rel_cache, size = self._lru.get(name)
abs_cache = os.path.join(self.cache_dir, rel_cache)
if fs.isdir(abs_cache):
logging.info('- reusing %r; size was %d', rel_cache,
size)
file_path.ensure_tree(os.path.dirname(dst))
self._sudo_chown(abs_cache)
fs.rename(abs_cache, dst)
self._remove(name)
return size
logging.warning('- expected directory %r, does not exist',
rel_cache)
self._remove(name)
# The named cache does not exist, create an empty directory. When
# uninstalling, we will move it back to the cache and create an an
# entry.
logging.info('- creating new directory')
file_path.ensure_tree(dst)
return 0
except (IOError, OSError) as ex:
# Raise using the original traceback.
exc = NamedCacheError(
'cannot install cache named %r at %r: %s' %
(name, dst, ex))
six.reraise(type(exc), exc, sys.exc_info()[2])
finally:
self._save()
def test_invalid_state(self):
file_path.ensure_tree(self.cache_dir())
statefile = os.path.join(
self.cache_dir(),
local_caching.DiskContentAddressedCache.STATE_FILE)
with open(statefile, 'w') as f:
f.write('invalid')
with open(os.path.join(self.cache_dir(), 'invalid'), 'w') as f:
f.write('invalid')
_ = self.get_cache(_get_policies())
self.assertEqual(fs.listdir(self.cache_dir()), ['state.json'])
def recreate_tree(outdir, indir, infiles, action, as_hash):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: One of accepted action of file_path.link_file().
as_hash: Output filename is the hash instead of relfile.
"""
logging.info(
'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
(outdir, indir, len(infiles), action, as_hash))
assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
if not os.path.isdir(outdir):
logging.info('Creating %s' % outdir)
fs.makedirs(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_hash:
# Do the hashtable specific checks.
if 'l' in metadata:
# Skip links when storing a hashtable.
continue
outfile = os.path.join(outdir, metadata['h'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if not 's' in metadata:
raise isolated_format.MappingError(
'Misconfigured item %s: %s' % (relfile, metadata))
if metadata['s'] == fs.stat(outfile).st_size:
continue
else:
logging.warn('Overwritting %s' % metadata['h'])
fs.remove(outfile)
else:
outfile = os.path.join(outdir, relfile)
file_path.ensure_tree(os.path.dirname(outfile))
if 'l' in metadata:
pointed = metadata['l']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
# symlink doesn't exist on Windows.
fs.symlink(pointed, outfile) # pylint: disable=E1101
else:
file_path.link_file(outfile, infile, action)
def file_write(path, content_generator):
"""Writes file content as generated by content_generator.
Creates the intermediary directory as needed.
Returns the number of bytes written.
Meant to be mocked out in unit tests.
"""
file_path.ensure_tree(os.path.dirname(path))
total = 0
with fs.open(path, 'wb') as f:
for d in content_generator:
total += len(d)
f.write(d)
return total
def uninstall(self, path, name):
"""Moves the cache directory back. Opposite to install().
NamedCache must be open. path must be absolute and unicode.
Raises Error if cannot uninstall the cache.
"""
logging.info('Uninstalling named cache %r from %r', name, path)
try:
_check_abs(path)
if not os.path.isdir(path):
logging.warning(
'Directory %r does not exist anymore. Cache lost.', path)
return
rel_cache = self._lru.get(name)
if rel_cache:
# Do not crash because cache already exists.
logging.warning('overwriting an existing named cache %r', name)
create_named_link = False
else:
rel_cache = self._allocate_dir()
create_named_link = True
# Move the dir and create an entry for the named cache.
abs_cache = os.path.join(self.root_dir, rel_cache)
logging.info('Moving %r to %r', path, abs_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(path, abs_cache)
self._lru.add(name, rel_cache)
if create_named_link:
# Create symlink <root_dir>/<named>/<name> -> <root_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
fs.symlink(abs_cache, named_path)
logging.info('Created symlink %r to %r', named_path, abs_cache)
except (OSError, Error) as ex:
raise Error(
'cannot uninstall cache named %r at %r: %s' % (
name, path, ex))
def prepare_logging(filename, root=None):
"""Prepare logging for scripts.
Makes it log in UTC all the time. Prepare a rotating file based log.
"""
assert not find_stderr(root)
formatter = UTCFormatter(
'%(process)d %(asctime)s %(severity)s: %(message)s')
# It is a requirement that the root logger is set to DEBUG, so the messages
# are not lost. It defaults to WARNING otherwise.
logger = root or logging.getLogger()
if not logger:
# Better print insanity than crash.
print >> sys.stderr, 'OMG NO ROOT'
return
logger.setLevel(logging.DEBUG)
stderr = logging.StreamHandler()
stderr.setFormatter(formatter)
stderr.addFilter(Filter())
# Default to ERROR.
stderr.setLevel(logging.ERROR)
logger.addHandler(stderr)
# Setup up logging to a constant file so we can debug issues where
# the results aren't properly sent to the result URL.
if filename:
file_path.ensure_tree(
os.path.dirname(os.path.abspath(unicode(filename))))
try:
rotating_file = NoInheritRotatingFileHandler(filename,
maxBytes=10 * 1024 *
1024,
backupCount=5,
encoding='utf-8')
rotating_file.setLevel(logging.DEBUG)
rotating_file.setFormatter(formatter)
rotating_file.addFilter(Filter())
logger.addHandler(rotating_file)
except Exception:
# May happen on cygwin. Do not crash.
logging.exception('Failed to open %s', filename)
def install(self, path, name):
"""Moves the directory for the specified named cache to |path|.
path must be absolute, unicode and must not exist.
Raises NamedCacheError if cannot install the cache.
"""
logging.info('Installing named cache %r to %r', name, path)
with self._lock:
try:
if os.path.isdir(path):
raise NamedCacheError(
'installation directory %r already exists' % path)
link_name = os.path.join(self.cache_dir, name)
if fs.exists(link_name):
fs.rmtree(link_name)
if name in self._lru:
rel_cache, _size = self._lru.get(name)
abs_cache = os.path.join(self.cache_dir, rel_cache)
if os.path.isdir(abs_cache):
logging.info('Moving %r to %r', abs_cache, path)
file_path.ensure_tree(os.path.dirname(path))
fs.rename(abs_cache, path)
self._remove(name)
return
logging.warning(
'directory for named cache %r does not exist at %s',
name, rel_cache)
self._remove(name)
# The named cache does not exist, create an empty directory.
# When uninstalling, we will move it back to the cache and create an
# an entry.
file_path.ensure_tree(path)
except (IOError, OSError) as ex:
raise NamedCacheError(
'cannot install cache named %r at %r: %s' %
(name, path, ex))
finally:
self._save()
def create_symlinks(self, root, named_caches):
"""Creates symlinks in |root| for specified named_caches.
named_caches must be a list of (name, path) tuples.
Requires NamedCache to be open.
Raises Error if cannot create a symlink.
"""
self._lock.assert_locked()
for name, path in named_caches:
try:
if os.path.isabs(path):
raise Error('named cache path must not be absolute')
if '..' in path.split(os.path.sep):
raise Error('named cache path must not contain ".."')
symlink_path = os.path.abspath(os.path.join(root, path))
file_path.ensure_tree(os.path.dirname(symlink_path))
fs.symlink(self.request(name), symlink_path)
except (OSError, Error) as ex:
raise Error(
'cannot create a symlink for cache named "%s" at "%s": %s' % (
name, symlink_path, ex))
def create_symlinks(self, root, named_caches):
"""Creates symlinks in |root| for the specified named_caches.
named_caches must be a list of (name, path) tuples.
Requires NamedCache to be open.
Raises Error if cannot create a symlink.
"""
self._lock.assert_locked()
for name, path in named_caches:
logging.info('Named cache %r -> %r', name, path)
try:
_validate_named_cache_path(path)
symlink_path = os.path.abspath(os.path.join(root, path))
file_path.ensure_tree(os.path.dirname(symlink_path))
requested = self.request(name)
logging.info('Symlink %r to %r', symlink_path, requested)
fs.symlink(requested, symlink_path)
except (OSError, Error) as ex:
raise Error(
'cannot create a symlink for cache named "%s" at "%s": %s'
% (name, symlink_path, ex))
def uninstall(self, src, name):
"""Moves the cache directory back into the named cache hive for an eventual
reuse.
The opposite of install().
src must be absolute and unicode. Its content is moved back into the local
named caches cache.
Returns the named cache size in bytes.
Raises NamedCacheError if cannot uninstall the cache.
"""
logging.info('NamedCache.uninstall(%r, %r)', src, name)
with self._lock:
try:
if not fs.isdir(src):
logging.warning(
'NamedCache: Directory %r does not exist anymore. Cache lost.',
src)
return
if name in self._lru:
# This shouldn't happen but just remove the preexisting one and move
# on.
logging.error('- overwriting existing cache!')
self._remove(name)
# Calculate the size of the named cache to keep. It's important because
# if size is zero (it's empty), we do not want to add it back to the
# named caches cache.
size = _get_recursive_size(src)
logging.info('- Size is %d', size)
if not size:
# Do not save empty named cache.
return size
# Move the dir and create an entry for the named cache.
rel_cache = self._allocate_dir()
abs_cache = os.path.join(self.cache_dir, rel_cache)
logging.info('- Moving to %r', rel_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(src, abs_cache)
self._lru.add(name, (rel_cache, size))
self._added.append(size)
# Create symlink <cache_dir>/<named>/<name> -> <cache_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if fs.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
try:
fs.symlink(os.path.join(u'..', rel_cache), named_path)
logging.info('NamedCache: Created symlink %r to %r',
named_path, abs_cache)
except OSError:
# Ignore on Windows. It happens when running as a normal user or when
# UAC is enabled and the user is a filtered administrator account.
if sys.platform != 'win32':
raise
return size
except (IOError, OSError) as ex:
# Raise using the original traceback.
exc = NamedCacheError(
'cannot uninstall cache named %r at %r: %s' %
(name, src, ex))
six.reraise(exc, None, sys.exc_info()[2])
finally:
# Call save() at every uninstall. The assumptions are:
# - The total the number of named caches is low, so the state.json file
# is small, so the time it takes to write it to disk is short.
# - The number of mapped named caches per task is low, so the number of
# times save() is called on tear-down isn't high enough to be
# significant.
# - uninstall() sometimes throws due to file locking on Windows or
# access rights on Linux. We want to keep as many as possible.
self._save()
def _save(self):
self._lock.assert_locked()
file_path.ensure_tree(self.cache_dir)
self._lru.save(self.state_file)
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'versions')),
isolateserver.CachePolicies(0, 0, 300),
hashlib.sha1,
trim=True)
with version_cache:
version_cache.cleanup()
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except isolateserver.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = isolateserver.DiskCache(
unicode(os.path.join(cache_dir, 'clients')),
isolateserver.CachePolicies(0, 0, 5),
hashlib.sha1,
trim=True)
with instance_cache:
instance_cache.cleanup()
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name,
instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url,
timeoutfn)
# A single host can run multiple swarming bots, but ATM they do not share
# same root bot directory. Thus, it is safe to use the same name for the
# binary.
cipd_bin_dir = unicode(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
file_path.remove(binary_path)
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
def run_tha_test(command, isolated_hash, storage, isolate_cache, outputs,
init_named_caches, leak_temp_dir, result_json, root_dir,
hard_timeout, grace_period, bot_file, install_packages_fn,
use_symlinks):
"""Runs an executable and records execution metadata.
Either command or isolated_hash must be specified.
If isolated_hash is specified, downloads the dependencies in the cache,
hardlinks them into a temporary directory and runs the command specified in
the .isolated.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
command: a list of string; the command to run OR optional arguments to add
to the command stated in the .isolated file if a command was
specified.
isolated_hash: the SHA-1 of the .isolated file that must be retrieved to
recreate the tree of files to run the target executable.
The command specified in the .isolated is executed.
Mutually exclusive with command argument.
storage: an isolateserver.Storage object to retrieve remote objects. This
object has a reference to an isolateserver.StorageApi, which does
the actual I/O.
isolate_cache: an isolateserver.LocalCache to keep from retrieving the
same objects constantly by caching the objects retrieved.
Can be on-disk or in-memory.
init_named_caches: a function (run_dir) => context manager that creates
symlinks for named caches in |run_dir|.
leak_temp_dir: if true, the temporary directory will be deliberately leaked
for later examination.
result_json: file path to dump result metadata into. If set, the process
exit code is always 0 unless an internal error occurred.
root_dir: path to the directory to use to create the temporary directory. If
not specified, a random temporary directory is created.
hard_timeout: kills the process if it lasts more than this amount of
seconds.
grace_period: number of seconds to wait between SIGTERM and SIGKILL.
install_packages_fn: context manager dir => CipdInfo, see
install_client_and_packages.
use_symlinks: create tree with symlinks instead of hardlinks.
Returns:
Process exit code that should be used.
"""
if result_json:
# Write a json output file right away in case we get killed.
result = {
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': 'Was terminated before completion',
'outputs_ref': None,
'version': 5,
}
tools.write_json(result_json, result, dense=True)
# run_isolated exit code. Depends on if result_json is used or not.
result = map_and_run(command, isolated_hash, storage, isolate_cache,
outputs, init_named_caches, leak_temp_dir, root_dir,
hard_timeout, grace_period, bot_file,
install_packages_fn, use_symlinks, True)
logging.info('Result:\n%s', tools.format_json(result, dense=True))
if result_json:
# We've found tests to delete 'work' when quitting, causing an exception
# here. Try to recreate the directory if necessary.
file_path.ensure_tree(os.path.dirname(result_json))
tools.write_json(result_json, result, dense=True)
# Only return 1 if there was an internal error.
return int(bool(result['internal_failure']))
# Marshall into old-style inline output.
if result['outputs_ref']:
data = {
'hash': result['outputs_ref']['isolated'],
'namespace': result['outputs_ref']['namespace'],
'storage': result['outputs_ref']['isolatedserver'],
}
sys.stdout.flush()
print('[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
tools.format_json(data, dense=True))
sys.stdout.flush()
return result['exit_code'] or int(bool(result['internal_failure']))
def get_client(service_url,
package_template,
version,
cache_dir,
timeout=None):
"""Returns a context manager that yields a CipdClient. A blocking call.
Upon exit from the context manager, the client binary may be deleted
(if the internal cache is full).
Args:
service_url (str): URL of the CIPD backend.
package_template (str): package name template of the CIPD client.
version (str): version of CIPD client package.
cache_dir: directory to store instance cache, version cache
and a hardlink to the client binary.
timeout (int): if not None, timeout in seconds for this function.
Yields:
CipdClient.
Raises:
Error if CIPD client version cannot be resolved or client cannot be fetched.
"""
timeoutfn = tools.sliding_timeout(timeout)
# Package names are always lower case.
# TODO(maruel): Assert instead?
package_name = package_template.lower().replace('${platform}',
get_platform())
# Resolve version to instance id.
# Is it an instance id already? They look like HEX SHA1.
if isolated_format.is_valid_hash(version, hashlib.sha1):
instance_id = version
elif ':' in version: # it's an immutable tag, cache the resolved version
# version_cache is {hash(package_name, tag) -> instance id} mapping.
# It does not take a lot of disk space.
version_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'versions')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=300,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
# Convert (package_name, version) to a string that may be used as a
# filename in disk cache by hashing it.
version_digest = hashlib.sha1('%s\n%s' %
(package_name, version)).hexdigest()
try:
with version_cache.getfileobj(version_digest) as f:
instance_id = f.read()
except local_caching.CacheMiss:
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
version_cache.write(version_digest, instance_id)
version_cache.trim()
else: # it's a ref, hit the backend
instance_id = resolve_version(service_url,
package_name,
version,
timeout=timeoutfn())
# instance_cache is {instance_id -> client binary} mapping.
# It is bounded by 5 client versions.
instance_cache = local_caching.DiskContentAddressedCache(
six.text_type(os.path.join(cache_dir, 'clients')),
local_caching.CachePolicies(
# 1GiB.
max_cache_size=1024 * 1024 * 1024,
min_free_space=0,
max_items=10,
# 3 weeks.
max_age_secs=21 * 24 * 60 * 60),
trim=True)
if instance_id not in instance_cache:
logging.info('Fetching CIPD client %s:%s', package_name, instance_id)
fetch_url = get_client_fetch_url(service_url,
package_name,
instance_id,
timeout=timeoutfn())
_fetch_cipd_client(instance_cache, instance_id, fetch_url, timeoutfn)
# A single host can run multiple swarming bots, but they cannot share same
# root bot directory. Thus, it is safe to use the same name for the binary.
cipd_bin_dir = six.text_type(os.path.join(cache_dir, 'bin'))
binary_path = os.path.join(cipd_bin_dir, 'cipd' + EXECUTABLE_SUFFIX)
if fs.isfile(binary_path):
# TODO(maruel): Do not unconditionally remove the binary.
try:
file_path.remove(binary_path)
except WindowsError: # pylint: disable=undefined-variable
# See whether cipd.exe is running for crbug.com/1028781
ret = subprocess42.call(['tasklist.exe'])
if ret:
logging.error('tasklist returns non-zero: %d', ret)
raise
else:
file_path.ensure_tree(cipd_bin_dir)
with instance_cache.getfileobj(instance_id) as f:
isolateserver.putfile(f, binary_path, 0o511) # -r-x--x--x
_ensure_batfile(binary_path)
yield CipdClient(binary_path,
package_name=package_name,
instance_id=instance_id,
service_url=service_url)
instance_cache.trim()
def uninstall(self, path, name):
"""Moves the cache directory back. Opposite to install().
path must be absolute and unicode.
Raises NamedCacheError if cannot uninstall the cache.
"""
logging.info('Uninstalling named cache %r from %r', name, path)
with self._lock:
try:
if not os.path.isdir(path):
logging.warning(
'Directory %r does not exist anymore. Cache lost.',
path)
return
if name in self._lru:
# This shouldn't happen but just remove the preexisting one and move
# on.
logging.warning('overwriting an existing named cache %r',
name)
self._remove(name)
rel_cache = self._allocate_dir()
# Move the dir and create an entry for the named cache.
abs_cache = os.path.join(self.cache_dir, rel_cache)
logging.info('Moving %r to %r', path, abs_cache)
file_path.ensure_tree(os.path.dirname(abs_cache))
fs.rename(path, abs_cache)
# That succeeded, calculate its new size.
size = _get_recursive_size(abs_cache)
if not size:
# Do not save empty named cache.
return
self._lru.add(name, (rel_cache, size))
self._added.append(size)
# Create symlink <cache_dir>/<named>/<name> -> <cache_dir>/<short name>
# for user convenience.
named_path = self._get_named_path(name)
if os.path.exists(named_path):
file_path.remove(named_path)
else:
file_path.ensure_tree(os.path.dirname(named_path))
try:
fs.symlink(abs_cache, named_path)
logging.info('Created symlink %r to %r', named_path,
abs_cache)
except OSError:
# Ignore on Windows. It happens when running as a normal user or when
# UAC is enabled and the user is a filtered administrator account.
if sys.platform != 'win32':
raise
except (IOError, OSError) as ex:
raise NamedCacheError(
'cannot uninstall cache named %r at %r: %s' %
(name, path, ex))
finally:
self._save()