def main():
"""CLI frontend to validate arguments."""
tools.disable_buffering()
parser = run_test_cases.OptionParserWithTestShardingAndFiltering(
usage='%prog <options> [gtest]')
# Override default seed value to default to 0.
parser.set_defaults(seed=0)
options, args = parser.parse_args()
if not args:
parser.error('Please provide the executable to run')
cmd = tools.fix_python_path(args)
try:
tests = run_test_cases.chromium_list_test_cases(
cmd,
os.getcwd(),
index=options.index,
shards=options.shards,
seed=options.seed,
disabled=options.disabled,
fails=options.fails,
flaky=options.flaky,
pre=False,
manual=options.manual)
for test in tests:
print test
except run_test_cases.Failure, e:
print e.args[0]
return e.args[1]
def main():
"""CLI frontend to validate arguments."""
tools.disable_buffering()
parser = run_test_cases.OptionParserWithTestShardingAndFiltering(
usage='%prog <options> [gtest]')
# Override default seed value to default to 0.
parser.set_defaults(seed=0)
options, args = parser.parse_args()
if not args:
parser.error('Please provide the executable to run')
cmd = tools.fix_python_path(args)
try:
tests = run_test_cases.chromium_list_test_cases(
cmd,
os.getcwd(),
index=options.index,
shards=options.shards,
seed=options.seed,
disabled=options.disabled,
fails=options.fails,
flaky=options.flaky,
pre=False,
manual=options.manual)
for test in tests:
print test
except run_test_cases.Failure, e:
print e.args[0]
return e.args[1]
def main():
"""CLI frontend to validate arguments."""
tools.disable_buffering()
parser = run_test_cases.OptionParserTestCases(
usage='%prog <options> [gtest]')
parser.format_description = lambda *_: parser.description
parser.add_option(
'-o', '--out',
help='output file, defaults to <executable>.test_cases')
parser.add_option(
'-r', '--root-dir',
help='Root directory under which file access should be noted')
parser.add_option(
'--trace-blacklist', action='append', default=[],
help='List of regexp to use as blacklist filter')
# TODO(maruel): Add support for options.timeout.
parser.remove_option('--timeout')
options, args = parser.parse_args()
if not args:
parser.error(
'Please provide the executable line to run, if you need fancy things '
'like xvfb, start this script from *inside* xvfb, it\'ll be much faster'
'.')
cmd = tools.fix_python_path(args)
cmd[0] = os.path.abspath(cmd[0])
if not os.path.isfile(cmd[0]):
parser.error('Tracing failed for: %s\nIt doesn\'t exit' % ' '.join(cmd))
if not options.out:
options.out = '%s.test_cases' % cmd[-1]
options.out = os.path.abspath(options.out)
if options.root_dir:
options.root_dir = os.path.abspath(options.root_dir)
logname = options.out + '.log'
test_cases = parser.process_gtest_options(cmd, os.getcwd(), options)
# Then run them.
print('Tracing...')
results = trace_test_cases(
cmd,
os.getcwd(),
test_cases,
options.jobs,
logname)
print('Reading trace logs...')
blacklist = tools.gen_blacklist(options.trace_blacklist)
write_details(logname, options.out, options.root_dir, blacklist, results)
return 0
def process_args(argv):
parser = OptionParserTestCases(
usage='%prog <options> [gtest]',
verbose=int(os.environ.get('ISOLATE_DEBUG', 0)))
parser.add_option(
'--run-all',
action='store_true',
help='Do not fail early when a large number of test cases fail')
parser.add_option(
'--max-failures', type='int',
help='Limit the number of failures before aborting')
parser.add_option(
'--retries', type='int', default=2,
help='Number of times each test case should be retried in case of '
'failure.')
parser.add_option(
'--no-dump',
action='store_true',
help='do not generate a .run_test_cases file')
parser.add_option(
'--no-cr',
action='store_true',
help='Use LF instead of CR for status progress')
parser.add_option(
'--result',
help='Override the default name of the generated .run_test_cases file')
group = optparse.OptionGroup(parser, 'google-test compability flags')
group.add_option(
'--gtest_list_tests',
action='store_true',
help='List all the test cases unformatted. Keeps compatibility with the '
'executable itself.')
group.add_option(
'--gtest_output',
default=os.environ.get('GTEST_OUTPUT', ''),
help='XML output to generate')
parser.add_option_group(group)
options, args = parser.parse_args(argv)
if not args:
parser.error(
'Please provide the executable line to run, if you need fancy things '
'like xvfb, start this script from *inside* xvfb, it\'ll be much faster'
'.')
if options.run_all and options.max_failures is not None:
parser.error('Use only one of --run-all or --max-failures')
return parser, options, tools.fix_python_path(args)
def CMDrun(parser, args):
"""Runs the test executable in an isolated (temporary) directory.
All the dependencies are mapped into the temporary directory and the
directory is cleaned up after the target exits.
Argument processing stops at -- and these arguments are appended to the
command line of the target to run. For example, use:
isolate.py run --isolated foo.isolated -- --gtest_filter=Foo.Bar
"""
add_isolate_options(parser)
add_skip_refresh_option(parser)
options, args = parser.parse_args(args)
process_isolate_options(parser, options, require_isolated=False)
complete_state = load_complete_state(options, os.getcwd(), None, options.skip_refresh)
cmd = complete_state.saved_state.command + args
if not cmd:
raise ExecutionError("No command to run.")
cmd = tools.fix_python_path(cmd)
outdir = run_isolated.make_temp_dir(u"isolate-%s" % datetime.date.today(), os.path.dirname(complete_state.root_dir))
try:
# TODO(maruel): Use run_isolated.run_tha_test().
cwd = create_isolate_tree(
outdir,
complete_state.root_dir,
complete_state.saved_state.files,
complete_state.saved_state.relative_cwd,
complete_state.saved_state.read_only,
)
file_path.ensure_command_has_abs_path(cmd, cwd)
logging.info("Running %s, cwd=%s" % (cmd, cwd))
try:
result = subprocess.call(cmd, cwd=cwd)
except OSError:
sys.stderr.write(
"Failed to executed the command; executable is missing, maybe you\n"
"forgot to map it in the .isolate file?\n %s\n in %s\n" % (" ".join(cmd), cwd)
)
result = 1
finally:
file_path.rmtree(outdir)
if complete_state.isolated_filepath:
complete_state.save_files()
return result
def CMDrun(parser, args):
"""Runs the test executable in an isolated (temporary) directory.
All the dependencies are mapped into the temporary directory and the
directory is cleaned up after the target exits.
Argument processing stops at -- and these arguments are appended to the
command line of the target to run. For example, use:
isolate.py run --isolated foo.isolated -- --gtest_filter=Foo.Bar
"""
add_isolate_options(parser)
add_skip_refresh_option(parser)
options, args = parser.parse_args(args)
process_isolate_options(parser, options, require_isolated=False)
complete_state = load_complete_state(
options, os.getcwd(), None, options.skip_refresh)
cmd = complete_state.saved_state.command + args
if not cmd:
raise ExecutionError('No command to run.')
cmd = tools.fix_python_path(cmd)
outdir = run_isolated.make_temp_dir(
u'isolate-%s' % datetime.date.today(),
os.path.dirname(complete_state.root_dir))
try:
# TODO(maruel): Use run_isolated.run_tha_test().
cwd = create_isolate_tree(
outdir, complete_state.root_dir, complete_state.saved_state.files,
complete_state.saved_state.relative_cwd,
complete_state.saved_state.read_only)
file_path.ensure_command_has_abs_path(cmd, cwd)
logging.info('Running %s, cwd=%s' % (cmd, cwd))
try:
result = subprocess.call(cmd, cwd=cwd)
except OSError:
sys.stderr.write(
'Failed to executed the command; executable is missing, maybe you\n'
'forgot to map it in the .isolate file?\n %s\n in %s\n' %
(' '.join(cmd), cwd))
result = 1
finally:
file_path.rmtree(outdir)
if complete_state.isolated_filepath:
complete_state.save_files()
return result
def safely_load_isolated(parser, options):
"""Loads a .isolated.state to extract the executable information.
Returns the CompleteState instance, the command and the list of test cases.
"""
config = isolate.CompleteState.load_files(options.isolated)
logging.debug(
'root_dir: %s relative_cwd: %s isolated: %s',
config.root_dir, config.saved_state.relative_cwd, options.isolated)
reldir = os.path.join(config.root_dir, config.saved_state.relative_cwd)
command = config.saved_state.command
test_cases = []
if command:
command = tools.fix_python_path(command)
test_xvfb(command, reldir)
test_cases = parser.process_gtest_options(command, reldir, options)
return config, command, test_cases
def safely_load_isolated(parser, options):
"""Loads a .isolated.state to extract the executable information.
Returns the CompleteState instance, the command and the list of test cases.
"""
config = isolate.CompleteState.load_files(options.isolated)
logging.debug('root_dir: %s relative_cwd: %s isolated: %s',
config.root_dir, config.saved_state.relative_cwd,
options.isolated)
reldir = os.path.join(config.root_dir, config.saved_state.relative_cwd)
command = config.saved_state.command
test_cases = []
if command:
command = tools.fix_python_path(command)
test_xvfb(command, reldir)
test_cases = parser.process_gtest_options(command, reldir, options)
return config, command, test_cases
def test_xvfb(command, rel_dir):
"""Calls back ourself if not running inside Xvfb and it's on the command line
to run.
Otherwise the X session will die while trying to start too many Xvfb
instances.
"""
if os.environ.get('_CHROMIUM_INSIDE_XVFB') == '1':
return
for index, item in enumerate(command):
if item.endswith('xvfb.py'):
# Note this has inside knowledge about src/testing/xvfb.py.
print('Restarting itself under Xvfb')
prefix = command[index:index+2]
prefix[0] = os.path.normpath(os.path.join(rel_dir, prefix[0]))
prefix[1] = os.path.normpath(os.path.join(rel_dir, prefix[1]))
cmd = tools.fix_python_path(prefix + sys.argv)
sys.exit(subprocess.call(cmd))
def test_xvfb(command, rel_dir):
"""Calls back ourself if not running inside Xvfb and it's on the command line
to run.
Otherwise the X session will die while trying to start too many Xvfb
instances.
"""
if os.environ.get('_CHROMIUM_INSIDE_XVFB') == '1':
return
for index, item in enumerate(command):
if item.endswith('xvfb.py'):
# Note this has inside knowledge about src/testing/xvfb.py.
print('Restarting itself under Xvfb')
prefix = command[index:index + 2]
prefix[0] = os.path.normpath(os.path.join(rel_dir, prefix[0]))
prefix[1] = os.path.normpath(os.path.join(rel_dir, prefix[1]))
cmd = tools.fix_python_path(prefix + sys.argv)
sys.exit(subprocess.call(cmd))
def main():
tools.disable_buffering()
parser = optparse.OptionParser(usage='%prog <options> [gtest]')
parser.disable_interspersed_args()
parser.add_option(
'-I', '--index',
type='int',
default=os.environ.get('GTEST_SHARD_INDEX'),
help='Shard index to run')
parser.add_option(
'-S', '--shards',
type='int',
default=os.environ.get('GTEST_TOTAL_SHARDS'),
help='Total number of shards to calculate from the --index to run')
options, args = parser.parse_args()
env = os.environ.copy()
env['GTEST_TOTAL_SHARDS'] = str(options.shards)
env['GTEST_SHARD_INDEX'] = str(options.index)
return subprocess.call(tools.fix_python_path(args), env=env)
def main():
tools.disable_buffering()
parser = optparse.OptionParser(usage='%prog <options> [gtest]')
parser.disable_interspersed_args()
parser.add_option('-I',
'--index',
type='int',
default=os.environ.get('GTEST_SHARD_INDEX'),
help='Shard index to run')
parser.add_option(
'-S',
'--shards',
type='int',
default=os.environ.get('GTEST_TOTAL_SHARDS'),
help='Total number of shards to calculate from the --index to run')
options, args = parser.parse_args()
env = os.environ.copy()
env['GTEST_TOTAL_SHARDS'] = str(options.shards)
env['GTEST_SHARD_INDEX'] = str(options.index)
return subprocess.call(tools.fix_python_path(args), env=env)
def map_and_run(command, isolated_hash, storage, isolate_cache, outputs,
init_named_caches, leak_temp_dir, root_dir, hard_timeout,
grace_period, bot_file, install_packages_fn, use_symlinks,
constant_run_path):
"""Runs a command with optional isolated input/output.
See run_tha_test for argument documentation.
Returns metadata about the result.
"""
assert isinstance(command, list), command
assert root_dir or root_dir is None
result = {
'duration': None,
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': None,
'stats': {
# 'isolated': {
# 'cipd': {
# 'duration': 0.,
# 'get_client_duration': 0.,
# },
# 'download': {
# 'duration': 0.,
# 'initial_number_items': 0,
# 'initial_size': 0,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# 'upload': {
# 'duration': 0.,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# },
},
# 'cipd_pins': {
# 'packages': [
# {'package_name': ..., 'version': ..., 'path': ...},
# ...
# ],
# 'client_package': {'package_name': ..., 'version': ...},
# },
'outputs_ref': None,
'version': 5,
}
if root_dir:
file_path.ensure_tree(root_dir, 0700)
elif isolate_cache.cache_dir:
root_dir = os.path.dirname(isolate_cache.cache_dir)
# See comment for these constants.
# If root_dir is not specified, it is not constant.
# TODO(maruel): This is not obvious. Change this to become an error once we
# make the constant_run_path an exposed flag.
if constant_run_path and root_dir:
run_dir = os.path.join(root_dir, ISOLATED_RUN_DIR)
os.mkdir(run_dir)
else:
run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir)
# storage should be normally set but don't crash if it is not. This can happen
# as Swarming task can run without an isolate server.
out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None
tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir)
cwd = run_dir
try:
with install_packages_fn(run_dir) as cipd_info:
if cipd_info:
result['stats']['cipd'] = cipd_info.stats
result['cipd_pins'] = cipd_info.pins
if isolated_hash:
isolated_stats = result['stats'].setdefault('isolated', {})
bundle, isolated_stats['download'] = fetch_and_map(
isolated_hash=isolated_hash,
storage=storage,
cache=isolate_cache,
outdir=run_dir,
use_symlinks=use_symlinks)
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd))
# Inject the command
if bundle.command:
command = bundle.command + command
if not command:
# Handle this as a task failure, not an internal failure.
sys.stderr.write(
'<No command was specified!>\n'
'<Please secify a command when triggering your Swarming task>\n'
)
result['exit_code'] = 1
return result
# If we have an explicit list of files to return, make sure their
# directories exist now.
if storage and outputs:
isolateserver.create_directories(run_dir, outputs)
command = tools.fix_python_path(command)
command = process_command(command, out_dir, bot_file)
file_path.ensure_command_has_abs_path(command, cwd)
with init_named_caches(run_dir):
sys.stdout.flush()
start = time.time()
try:
result['exit_code'], result[
'had_hard_timeout'] = run_command(
command, cwd, get_command_env(tmp_dir, cipd_info),
hard_timeout, grace_period)
finally:
result['duration'] = max(time.time() - start, 0)
except Exception as e:
# An internal error occurred. Report accordingly so the swarming task will
# be retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
# Clean up
finally:
try:
# Try to link files to the output directory, if specified.
if out_dir:
link_outputs_to_outdir(run_dir, out_dir, outputs)
success = False
if leak_temp_dir:
success = True
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the run directory, thus failing the task.\n'
'This may be due to a subprocess outliving the main task\n'
'process, holding on to resources. Please fix the task so\n'
'that it releases resources and cleans up subprocesses.'
)
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the temp directory, thus failing the task.\n'
'This may be due to a subprocess outliving the main task\n'
'process, holding on to resources. Please fix the task so\n'
'that it releases resources and cleans up subprocesses.'
)
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
if out_dir:
isolated_stats = result['stats'].setdefault('isolated', {})
result['outputs_ref'], success, isolated_stats['upload'] = (
delete_and_upload(storage, out_dir, leak_temp_dir))
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
if out_dir:
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
def map_and_run(command, isolated_hash, storage, isolate_cache, outputs,
init_name_caches, leak_temp_dir, root_dir, hard_timeout,
grace_period, bot_file, extra_args, install_packages_fn,
use_symlinks):
"""Runs a command with optional isolated input/output.
See run_tha_test for argument documentation.
Returns metadata about the result.
"""
assert root_dir or root_dir is None
assert bool(command) ^ bool(isolated_hash)
result = {
'duration': None,
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': None,
'stats': {
# 'isolated': {
# 'cipd': {
# 'duration': 0.,
# 'get_client_duration': 0.,
# },
# 'download': {
# 'duration': 0.,
# 'initial_number_items': 0,
# 'initial_size': 0,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# 'upload': {
# 'duration': 0.,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# },
},
# 'cipd_pins': {
# 'packages': [
# {'package_name': ..., 'version': ..., 'path': ...},
# ...
# ],
# 'client_package': {'package_name': ..., 'version': ...},
# },
'outputs_ref': None,
'version': 5,
}
if root_dir:
file_path.ensure_tree(root_dir, 0700)
elif isolate_cache.cache_dir:
root_dir = os.path.dirname(isolate_cache.cache_dir)
# See comment for these constants.
run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir)
# storage should be normally set but don't crash if it is not. This can happen
# as Swarming task can run without an isolate server.
out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None
tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir)
cwd = run_dir
try:
cipd_info = install_packages_fn(run_dir)
if cipd_info:
result['stats']['cipd'] = cipd_info['stats']
result['cipd_pins'] = cipd_info['cipd_pins']
if isolated_hash:
isolated_stats = result['stats'].setdefault('isolated', {})
bundle, isolated_stats['download'] = fetch_and_map(
isolated_hash=isolated_hash,
storage=storage,
cache=isolate_cache,
outdir=run_dir,
use_symlinks=use_symlinks)
if not bundle.command:
# Handle this as a task failure, not an internal failure.
sys.stderr.write(
'<The .isolated doesn\'t declare any command to run!>\n'
'<Check your .isolate for missing \'command\' variable>\n')
if os.environ.get('SWARMING_TASK_ID'):
# Give an additional hint when running as a swarming task.
sys.stderr.write('<This occurs at the \'isolate\' step>\n')
result['exit_code'] = 1
return result
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd))
command = bundle.command + extra_args
# If we have an explicit list of files to return, make sure their
# directories exist now.
if storage and outputs:
isolateserver.create_directories(run_dir, outputs)
command = tools.fix_python_path(command)
command = process_command(command, out_dir, bot_file)
file_path.ensure_command_has_abs_path(command, cwd)
init_name_caches(run_dir)
sys.stdout.flush()
start = time.time()
try:
result['exit_code'], result['had_hard_timeout'] = run_command(
command, cwd, tmp_dir, hard_timeout, grace_period)
finally:
result['duration'] = max(time.time() - start, 0)
except Exception as e:
# An internal error occurred. Report accordingly so the swarming task will
# be retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
# Clean up
finally:
try:
# Try to link files to the output directory, if specified.
if out_dir:
link_outputs_to_outdir(run_dir, out_dir, outputs)
success = False
if leak_temp_dir:
success = True
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the run directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
if out_dir:
isolated_stats = result['stats'].setdefault('isolated', {})
result['outputs_ref'], success, isolated_stats['upload'] = (
delete_and_upload(storage, out_dir, leak_temp_dir))
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
if out_dir:
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
def map_and_run(data, constant_run_path):
"""Runs a command with optional isolated input/output.
Arguments:
- data: TaskData instance.
- constant_run_path: TODO
Returns metadata about the result.
"""
result = {
'duration': None,
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': 'run_isolated did not complete properly',
'stats': {
# 'isolated': {
# 'cipd': {
# 'duration': 0.,
# 'get_client_duration': 0.,
# },
# 'download': {
# 'duration': 0.,
# 'initial_number_items': 0,
# 'initial_size': 0,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# 'upload': {
# 'duration': 0.,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# },
},
# 'cipd_pins': {
# 'packages': [
# {'package_name': ..., 'version': ..., 'path': ...},
# ...
# ],
# 'client_package': {'package_name': ..., 'version': ...},
# },
'outputs_ref': None,
'version': 5,
}
if data.root_dir:
file_path.ensure_tree(data.root_dir, 0700)
elif data.isolate_cache.cache_dir:
data = data._replace(
root_dir=os.path.dirname(data.isolate_cache.cache_dir))
# See comment for these constants.
# If root_dir is not specified, it is not constant.
# TODO(maruel): This is not obvious. Change this to become an error once we
# make the constant_run_path an exposed flag.
if constant_run_path and data.root_dir:
run_dir = os.path.join(data.root_dir, ISOLATED_RUN_DIR)
if os.path.isdir(run_dir):
file_path.rmtree(run_dir)
os.mkdir(run_dir, 0700)
else:
run_dir = make_temp_dir(ISOLATED_RUN_DIR, data.root_dir)
# storage should be normally set but don't crash if it is not. This can happen
# as Swarming task can run without an isolate server.
out_dir = make_temp_dir(
ISOLATED_OUT_DIR, data.root_dir) if data.storage else None
tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, data.root_dir)
cwd = run_dir
if data.relative_cwd:
cwd = os.path.normpath(os.path.join(cwd, data.relative_cwd))
command = data.command
try:
with data.install_packages_fn(run_dir) as cipd_info:
if cipd_info:
result['stats']['cipd'] = cipd_info.stats
result['cipd_pins'] = cipd_info.pins
if data.isolated_hash:
isolated_stats = result['stats'].setdefault('isolated', {})
bundle, isolated_stats['download'] = fetch_and_map(
isolated_hash=data.isolated_hash,
storage=data.storage,
cache=data.isolate_cache,
outdir=run_dir,
use_symlinks=data.use_symlinks)
change_tree_read_only(run_dir, bundle.read_only)
# Inject the command
if not command and bundle.command:
command = bundle.command + data.extra_args
# Only set the relative directory if the isolated file specified a
# command, and no raw command was specified.
if bundle.relative_cwd:
cwd = os.path.normpath(os.path.join(cwd, bundle.relative_cwd))
if not command:
# Handle this as a task failure, not an internal failure.
sys.stderr.write(
'<No command was specified!>\n'
'<Please secify a command when triggering your Swarming task>\n')
result['exit_code'] = 1
return result
if not cwd.startswith(run_dir):
# Handle this as a task failure, not an internal failure. This is a
# 'last chance' way to gate against directory escape.
sys.stderr.write('<Relative CWD is outside of run directory!>\n')
result['exit_code'] = 1
return result
if not os.path.isdir(cwd):
# Accepts relative_cwd that does not exist.
os.makedirs(cwd, 0700)
# If we have an explicit list of files to return, make sure their
# directories exist now.
if data.storage and data.outputs:
isolateserver.create_directories(run_dir, data.outputs)
command = tools.fix_python_path(command)
command = process_command(command, out_dir, data.bot_file)
file_path.ensure_command_has_abs_path(command, cwd)
with data.install_named_caches(run_dir):
sys.stdout.flush()
start = time.time()
try:
# Need to switch the default account before 'get_command_env' call,
# so it can grab correct value of LUCI_CONTEXT env var.
with set_luci_context_account(data.switch_to_account, tmp_dir):
env = get_command_env(
tmp_dir, cipd_info, run_dir, data.env, data.env_prefix)
result['exit_code'], result['had_hard_timeout'] = run_command(
command, cwd, env, data.hard_timeout, data.grace_period)
finally:
result['duration'] = max(time.time() - start, 0)
# We successfully ran the command, set internal_failure back to
# None (even if the command failed, it's not an internal error).
result['internal_failure'] = None
except Exception as e:
# An internal error occurred. Report accordingly so the swarming task will
# be retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
# Clean up
finally:
try:
# Try to link files to the output directory, if specified.
if out_dir:
link_outputs_to_outdir(run_dir, out_dir, data.outputs)
success = False
if data.leak_temp_dir:
success = True
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
sys.stderr.write(OUTLIVING_ZOMBIE_MSG % ('run', data.grace_period))
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
sys.stderr.write(OUTLIVING_ZOMBIE_MSG % ('temp', data.grace_period))
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
if out_dir:
isolated_stats = result['stats'].setdefault('isolated', {})
result['outputs_ref'], success, isolated_stats['upload'] = (
delete_and_upload(data.storage, out_dir, data.leak_temp_dir))
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
if out_dir:
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
