def CMDremap(parser, args):
"""Creates a directory with all the dependencies mapped into it.
Useful to test manually why a test is failing. The target executable is not
run.
"""
add_isolate_options(parser)
add_outdir_options(parser)
add_skip_refresh_option(parser)
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
cwd = os.getcwd()
process_isolate_options(parser, options, cwd, require_isolated=False)
process_outdir_options(parser, options, cwd)
complete_state = load_complete_state(options, cwd, None, options.skip_refresh)
if not fs.isdir(options.outdir):
fs.makedirs(options.outdir)
print('Remapping into %s' % options.outdir)
if fs.listdir(options.outdir):
raise ExecutionError('Can\'t remap in a non-empty directory')
create_isolate_tree(
options.outdir, complete_state.root_dir, complete_state.saved_state.files,
complete_state.saved_state.relative_cwd,
complete_state.saved_state.read_only)
if complete_state.isolated_filepath:
complete_state.save_files()
return 0
def __init__(self, cache_dir, policies, time_fn=None):
"""Initializes NamedCaches.
Arguments:
- cache_dir is a directory for persistent cache storage.
- policies is a CachePolicies instance.
- time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested. Used in unit tests.
"""
super(NamedCache, self).__init__(cache_dir)
self._policies = policies
# LRU {cache_name -> tuple(cache_location, size)}
self.state_file = os.path.join(cache_dir, self.STATE_FILE)
self._lru = lru.LRUDict()
if not fs.isdir(self.cache_dir):
fs.makedirs(self.cache_dir)
elif os.path.isfile(self.state_file):
try:
self._lru = lru.LRUDict.load(self.state_file)
except ValueError:
logging.exception('failed to load named cache state file')
logging.warning('deleting named caches')
file_path.rmtree(self.cache_dir)
with self._lock:
self._try_upgrade()
if time_fn:
self._lru.time_fn = time_fn
def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
"""Creates a isolated tree usable for test execution.
Returns the current working directory where the isolated command should be
started in.
"""
# Forcibly copy when the tree has to be read only. Otherwise the inode is
# modified, and this cause real problems because the user's source tree
# becomes read only. On the other hand, the cost of doing file copy is huge.
if read_only not in (0, None):
action = file_path.COPY
else:
action = file_path.HARDLINK_WITH_FALLBACK
recreate_tree(
outdir=outdir,
indir=root_dir,
infiles=files,
action=action,
as_hash=False)
cwd = os.path.normpath(os.path.join(outdir, relative_cwd))
if not fs.isdir(cwd):
# It can happen when no files are mapped from the directory containing the
# .isolate file. But the directory must exist to be the current working
# directory.
fs.makedirs(cwd)
run_isolated.change_tree_read_only(outdir, read_only)
return cwd
def CMDremap(parser, args):
"""Creates a directory with all the dependencies mapped into it.
Useful to test manually why a test is failing. The target executable is not
run.
"""
add_isolate_options(parser)
add_outdir_options(parser)
add_skip_refresh_option(parser)
options, args = parser.parse_args(args)
if args:
parser.error('Unsupported argument: %s' % args)
cwd = os.getcwd()
process_isolate_options(parser, options, cwd, require_isolated=False)
process_outdir_options(parser, options, cwd)
complete_state = load_complete_state(options, cwd, None, options.skip_refresh)
if not fs.isdir(options.outdir):
fs.makedirs(options.outdir)
print('Remapping into %s' % options.outdir)
if fs.listdir(options.outdir):
raise ExecutionError('Can\'t remap in a non-empty directory')
create_isolate_tree(
options.outdir, complete_state.root_dir, complete_state.saved_state.files,
complete_state.saved_state.relative_cwd,
complete_state.saved_state.read_only)
if complete_state.isolated_filepath:
complete_state.save_files()
return 0
def _load(self, trim, time_fn):
"""Loads state of the cache from json file.
If cache_dir does not exist on disk, it is created.
"""
self._lock.assert_locked()
if not fs.isfile(self.state_file):
if not fs.isdir(self.cache_dir):
fs.makedirs(self.cache_dir)
else:
# Load state of the cache.
try:
self._lru = lru.LRUDict.load(self.state_file)
except ValueError as err:
logging.error('Failed to load cache state: %s' % (err, ))
# Don't want to keep broken state file.
file_path.try_remove(self.state_file)
if time_fn:
self._lru.time_fn = time_fn
if trim:
self._trim()
# We want the initial cache size after trimming, i.e. what is readily
# avaiable.
self._initial_number_items = len(self._lru)
self._initial_size = sum(self._lru.itervalues())
if self._evicted:
logging.info('Trimming evicted items with the following sizes: %s',
sorted(self._evicted))
def create_isolate_tree(outdir, root_dir, files, relative_cwd, read_only):
"""Creates a isolated tree usable for test execution.
Returns the current working directory where the isolated command should be
started in.
"""
# Forcibly copy when the tree has to be read only. Otherwise the inode is
# modified, and this cause real problems because the user's source tree
# becomes read only. On the other hand, the cost of doing file copy is huge.
if read_only not in (0, None):
action = file_path.COPY
else:
action = file_path.HARDLINK_WITH_FALLBACK
recreate_tree(
outdir=outdir,
indir=root_dir,
infiles=files,
action=action,
as_hash=False)
cwd = os.path.normpath(os.path.join(outdir, relative_cwd))
if not fs.isdir(cwd):
# It can happen when no files are mapped from the directory containing the
# .isolate file. But the directory must exist to be the current working
# directory.
fs.makedirs(cwd)
run_isolated.change_tree_read_only(outdir, read_only)
return cwd
def ensure_tree(path, perm=0o777):
"""Ensures a directory exists."""
if not fs.isdir(path):
try:
fs.makedirs(path, perm)
except OSError as e:
# Do not raise if directory exists.
if e.errno != errno.EEXIST or not fs.isdir(path):
raise
def recreate_tree(outdir, indir, infiles, action, as_hash):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: One of accepted action of file_path.link_file().
as_hash: Output filename is the hash instead of relfile.
"""
logging.info(
'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
(outdir, indir, len(infiles), action, as_hash))
assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
if not os.path.isdir(outdir):
logging.info('Creating %s' % outdir)
fs.makedirs(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_hash:
# Do the hashtable specific checks.
if 'l' in metadata:
# Skip links when storing a hashtable.
continue
outfile = os.path.join(outdir, metadata['h'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if not 's' in metadata:
raise isolated_format.MappingError(
'Misconfigured item %s: %s' % (relfile, metadata))
if metadata['s'] == fs.stat(outfile).st_size:
continue
else:
logging.warn('Overwritting %s' % metadata['h'])
fs.remove(outfile)
else:
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
fs.makedirs(outsubdir)
if 'l' in metadata:
pointed = metadata['l']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
# symlink doesn't exist on Windows.
fs.symlink(pointed, outfile) # pylint: disable=E1101
else:
file_path.link_file(outfile, infile, action)
def recreate_tree(outdir, indir, infiles, action, as_hash):
"""Creates a new tree with only the input files in it.
Arguments:
outdir: Output directory to create the files in.
indir: Root directory the infiles are based in.
infiles: dict of files to map from |indir| to |outdir|.
action: One of accepted action of file_path.link_file().
as_hash: Output filename is the hash instead of relfile.
"""
logging.info(
'recreate_tree(outdir=%s, indir=%s, files=%d, action=%s, as_hash=%s)' %
(outdir, indir, len(infiles), action, as_hash))
assert os.path.isabs(outdir) and outdir == os.path.normpath(outdir), outdir
if not os.path.isdir(outdir):
logging.info('Creating %s' % outdir)
fs.makedirs(outdir)
for relfile, metadata in infiles.iteritems():
infile = os.path.join(indir, relfile)
if as_hash:
# Do the hashtable specific checks.
if 'l' in metadata:
# Skip links when storing a hashtable.
continue
outfile = os.path.join(outdir, metadata['h'])
if os.path.isfile(outfile):
# Just do a quick check that the file size matches. No need to stat()
# again the input file, grab the value from the dict.
if not 's' in metadata:
raise isolated_format.MappingError(
'Misconfigured item %s: %s' % (relfile, metadata))
if metadata['s'] == fs.stat(outfile).st_size:
continue
else:
logging.warn('Overwritting %s' % metadata['h'])
fs.remove(outfile)
else:
outfile = os.path.join(outdir, relfile)
outsubdir = os.path.dirname(outfile)
if not os.path.isdir(outsubdir):
fs.makedirs(outsubdir)
if 'l' in metadata:
pointed = metadata['l']
logging.debug('Symlink: %s -> %s' % (outfile, pointed))
# symlink doesn't exist on Windows.
fs.symlink(pointed, outfile) # pylint: disable=E1101
else:
file_path.link_file(outfile, infile, action)
def make_tree(out, contents):
for relpath, content in sorted(contents.items()):
filepath = os.path.join(out, relpath.replace('/', os.path.sep))
dirpath = os.path.dirname(filepath)
if not fs.isdir(dirpath):
fs.makedirs(dirpath, 0o700)
if isinstance(content, SymLink):
fs.symlink(content, filepath)
else:
mode = 0o700 if relpath.endswith('.py') else 0o600
flags = os.O_WRONLY | os.O_CREAT
if sys.platform == 'win32':
# pylint: disable=no-member
flags |= os.O_BINARY
with os.fdopen(os.open(filepath, flags, mode), 'wb') as f:
f.write(six.ensure_binary(content))
def __init__(self, task_output_dir, shard_count):
"""Initializes TaskOutputCollector, ensures |task_output_dir| exists.
Args:
task_output_dir: (optional) local directory to put fetched files to.
shard_count: expected number of task shards.
"""
self.task_output_dir = (
unicode(os.path.abspath(task_output_dir))
if task_output_dir else task_output_dir)
self.shard_count = shard_count
self._lock = threading.Lock()
self._per_shard_results = {}
self._storage = None
if self.task_output_dir and not fs.isdir(self.task_output_dir):
fs.makedirs(self.task_output_dir)
def _load(self, trim, time_fn):
"""Loads state of the cache from json file.
If cache_dir does not exist on disk, it is created.
"""
self._lock.assert_locked()
if not fs.isfile(self.state_file):
if not fs.isdir(self.cache_dir):
fs.makedirs(self.cache_dir)
else:
# Load state of the cache.
try:
self._lru = lru.LRUDict.load(self.state_file)
except ValueError as err:
logging.error('Failed to load cache state: %s' % (err, ))
# Don't want to keep broken state file.
file_path.try_remove(self.state_file)
if time_fn:
self._lru.time_fn = time_fn
if trim:
self._trim()
def test_load_corrupted_state(self):
# cleanup() handles a broken state file.
fs.mkdir(self.cache_dir)
c = local_caching.NamedCache
with fs.open(os.path.join(self.cache_dir, c.STATE_FILE), 'w') as f:
f.write('}}}}')
fs.makedirs(os.path.join(self.cache_dir, '1'), 0o777)
cache = self.get_cache(_get_policies())
self._add_one_item(cache, 1)
self.assertTrue(
fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertTrue(
fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertEqual([], cache.trim())
self.assertTrue(
fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertTrue(
fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1')))
self.assertEqual(True, cache.cleanup())
self.assertEqual(
sorted([cache.NAMED_DIR, cache.STATE_FILE, cache._lru[u'1'][0]]),
sorted(fs.listdir(cache.cache_dir)))
def __init__(self, cache_dir, policies, time_fn=None):
"""Initializes NamedCaches.
Arguments:
- cache_dir is a directory for persistent cache storage.
- policies is a CachePolicies instance.
- time_fn is a function that returns timestamp (float) and used to take
timestamps when new caches are requested. Used in unit tests.
"""
super(NamedCache, self).__init__(cache_dir)
self._policies = policies
# LRU {cache_name -> tuple(cache_location, size)}
self.state_file = os.path.join(cache_dir, self.STATE_FILE)
self._lru = lru.LRUDict()
if not fs.isdir(self.cache_dir):
fs.makedirs(self.cache_dir)
elif fs.isfile(self.state_file):
try:
self._lru = lru.LRUDict.load(self.state_file)
for _, size in self._lru.values():
if not isinstance(size, six.integer_types):
with open(self.state_file, 'r') as f:
logging.info('named cache state file: %s\n%s',
self.state_file, f.read())
raise ValueError("size is not integer: %s" % size)
except ValueError:
logging.exception(
'NamedCache: failed to load named cache state file; obliterating'
)
file_path.rmtree(self.cache_dir)
fs.makedirs(self.cache_dir)
self._lru = lru.LRUDict()
with self._lock:
self._try_upgrade()
if time_fn:
self._lru.time_fn = time_fn
def map_and_run(
isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout,
grace_period, extra_args):
"""Maps and run the command. Returns metadata about the result."""
# TODO(maruel): Include performance statistics.
result = {
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': None,
'outputs_ref': None,
'version': 2,
}
if root_dir:
if not fs.isdir(root_dir):
fs.makedirs(root_dir, 0700)
prefix = u''
else:
root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
prefix = u'isolated_'
run_dir = make_temp_dir(prefix + u'run', root_dir)
out_dir = make_temp_dir(prefix + u'out', root_dir)
tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
try:
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
result['exit_code'], result['had_hard_timeout'] = run_command(
process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
grace_period)
except Exception as e:
# An internal error occured. Report accordingly so the swarming task will be
# retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
finally:
try:
if leak_temp_dir:
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the run directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
result['outputs_ref'], success = delete_and_upload(
storage, out_dir, leak_temp_dir)
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
def ensure_tree(path, perm=0777):
"""Ensures a directory exists."""
if not fs.isdir(path):
fs.makedirs(path, perm)
def map_and_run(
isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout,
grace_period, extra_args):
"""Maps and run the command. Returns metadata about the result."""
result = {
'duration': None,
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': None,
'stats': {
# 'download': {
# 'duration': 0.,
# 'initial_number_items': 0,
# 'initial_size': 0,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# 'upload': {
# 'duration': 0.,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
},
'outputs_ref': None,
'version': 3,
}
if root_dir:
if not fs.isdir(root_dir):
fs.makedirs(root_dir, 0700)
prefix = u''
else:
root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
prefix = u'isolated_'
run_dir = make_temp_dir(prefix + u'run', root_dir)
out_dir = make_temp_dir(prefix + u'out', root_dir)
tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
try:
start = time.time()
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir)
if not bundle.command:
# Handle this as a task failure, not an internal failure.
sys.stderr.write(
'<The .isolated doesn\'t declare any command to run!>\n'
'<Check your .isolate for missing \'command\' variable>\n')
if os.environ.get('SWARMING_TASK_ID'):
# Give an additional hint when running as a swarming task.
sys.stderr.write('<This occurs at the \'isolate\' step>\n')
result['exit_code'] = 1
return result
result['stats']['download'] = {
'duration': time.time() - start,
'initial_number_items': cache.initial_number_items,
'initial_size': cache.initial_size,
'items_cold': base64.b64encode(large.pack(sorted(cache.added))),
'items_hot': base64.b64encode(
large.pack(sorted(set(cache.linked) - set(cache.added)))),
}
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
sys.stdout.flush()
start = time.time()
try:
result['exit_code'], result['had_hard_timeout'] = run_command(
process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
grace_period)
finally:
result['duration'] = max(time.time() - start, 0)
except Exception as e:
# An internal error occured. Report accordingly so the swarming task will be
# retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
finally:
try:
if leak_temp_dir:
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the run directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
start = time.time()
result['outputs_ref'], success, cold, hot = delete_and_upload(
storage, out_dir, leak_temp_dir)
result['stats']['upload'] = {
'duration': time.time() - start,
'items_cold': base64.b64encode(large.pack(cold)),
'items_hot': base64.b64encode(large.pack(hot)),
}
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
