def process_shard_result(self, shard_index, result):
"""Stores results of a single task shard, fetches output files if necessary.
Modifies |result| in place.
shard_index is 0-based.
Called concurrently from multiple threads.
"""
# Sanity check index is in expected range.
assert isinstance(shard_index, int)
if shard_index < 0 or shard_index >= self.shard_count:
logging.warning(
'Shard index %d is outside of expected range: [0; %d]',
shard_index, self.shard_count - 1)
return
if result.get('outputs_ref'):
ref = result['outputs_ref']
result['outputs_ref']['view_url'] = '%s/browse?%s' % (
ref['isolatedserver'],
urllib.urlencode(
[('namespace', ref['namespace']), ('hash', ref['isolated'])]))
# Store result dict of that shard, ignore results we've already seen.
with self._lock:
if shard_index in self._per_shard_results:
logging.warning('Ignoring duplicate shard index %d', shard_index)
return
self._per_shard_results[shard_index] = result
# Fetch output files if necessary.
if self.task_output_dir and result.get('outputs_ref'):
storage = self._get_storage(
result['outputs_ref']['isolatedserver'],
result['outputs_ref']['namespace'])
if storage:
# Output files are supposed to be small and they are not reused across
# tasks. So use MemoryCache for them instead of on-disk cache. Make
# files writable, so that calling script can delete them.
isolateserver.fetch_isolated(
result['outputs_ref']['isolated'],
storage,
isolateserver.MemoryCache(file_mode_mask=0700),
os.path.join(self.task_output_dir, str(shard_index)),
False)
def process_shard_result(self, shard_index, result):
"""Stores results of a single task shard, fetches output files if necessary.
Modifies |result| in place.
Called concurrently from multiple threads.
"""
# Sanity check index is in expected range.
assert isinstance(shard_index, int)
if shard_index < 0 or shard_index >= self.shard_count:
logging.warning(
'Shard index %d is outside of expected range: [0; %d]',
shard_index, self.shard_count - 1)
return
assert not 'isolated_out' in result
result['isolated_out'] = None
for output in result['outputs']:
isolated_files_location = extract_output_files_location(output)
if isolated_files_location:
if result['isolated_out']:
raise ValueError('Unexpected two task with output')
result['isolated_out'] = isolated_files_location
# Store result dict of that shard, ignore results we've already seen.
with self._lock:
if shard_index in self._per_shard_results:
logging.warning('Ignoring duplicate shard index %d', shard_index)
return
self._per_shard_results[shard_index] = result
# Fetch output files if necessary.
if self.task_output_dir and result['isolated_out']:
storage = self._get_storage(
result['isolated_out']['server'],
result['isolated_out']['namespace'])
if storage:
# Output files are supposed to be small and they are not reused across
# tasks. So use MemoryCache for them instead of on-disk cache. Make
# files writable, so that calling script can delete them.
isolateserver.fetch_isolated(
result['isolated_out']['hash'],
storage,
isolateserver.MemoryCache(file_mode_mask=0700),
os.path.join(self.task_output_dir, str(shard_index)),
False)
def process_shard_result(self, result):
"""Stores results of a single task shard, fetches output files if necessary.
Called concurrently from multiple threads.
"""
# We are going to put |shard_index| into a file path. Make sure it is int.
shard_index = result["config_instance_index"]
if not isinstance(shard_index, int):
raise ValueError("Shard index should be an int: %r" % (shard_index,))
# Sanity check index is in expected range.
if shard_index < 0 or shard_index >= self.shard_count:
logging.warning("Shard index %d is outside of expected range: [0; %d]", shard_index, self.shard_count - 1)
return
# Store result dict of that shard, ignore results we've already seen.
with self._lock:
if shard_index in self._per_shard_results:
logging.warning("Ignoring duplicate shard index %d", shard_index)
return
self._per_shard_results[shard_index] = result
# Fetch output files if necessary.
isolated_files_location = extract_output_files_location(result["output"])
if isolated_files_location:
isolate_server, namespace, isolated_hash = isolated_files_location
storage = self._get_storage(isolate_server, namespace)
if storage:
# Output files are supposed to be small and they are not reused across
# tasks. So use MemoryCache for them instead of on-disk cache. Make
# files writable, so that calling script can delete them.
isolateserver.fetch_isolated(
isolated_hash,
storage,
isolateserver.MemoryCache(file_mode_mask=0700),
os.path.join(self.task_output_dir, str(shard_index)),
False,
)
def fetch_and_map(isolated_hash, storage, cache, outdir, use_symlinks):
"""Fetches an isolated tree, create the tree and returns (bundle, stats)."""
start = time.time()
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=outdir,
use_symlinks=use_symlinks)
return bundle, {
'duration': time.time() - start,
'items_cold': base64.b64encode(large.pack(sorted(cache.added))),
'items_hot': base64.b64encode(
large.pack(sorted(set(cache.used) - set(cache.added)))),
}
def run_tha_test(isolated_hash, storage, cache, algo, outdir):
"""Downloads the dependencies in the cache, hardlinks them into a |outdir|
and runs the executable.
"""
try:
try:
settings = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
algo=algo,
outdir=outdir,
os_flavor=get_flavor(),
require_command=True)
except isolateserver.ConfigError as e:
print >> sys.stderr, str(e)
return 1
if settings.read_only:
logging.info('Making files read only')
make_writable(outdir, True)
cwd = os.path.normpath(os.path.join(outdir, settings.relative_cwd))
logging.info('Running %s, cwd=%s' % (settings.command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault('RUN_TEST_CASES_LOG_FILE',
os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
try:
with tools.Profiler('RunTest'):
return subprocess.call(settings.command, cwd=cwd, env=env)
except OSError:
print >> sys.stderr, 'Failed to run %s; cwd=%s' % (
settings.command, cwd)
return 1
finally:
if outdir:
rmtree(outdir)
def run_tha_test(isolated_hash, storage, cache, algo, outdir):
"""Downloads the dependencies in the cache, hardlinks them into a |outdir|
and runs the executable.
"""
try:
try:
settings = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
algo=algo,
outdir=outdir,
os_flavor=get_flavor(),
require_command=True)
except isolateserver.ConfigError as e:
print >> sys.stderr, str(e)
return 1
if settings.read_only:
logging.info('Making files read only')
make_writable(outdir, True)
cwd = os.path.normpath(os.path.join(outdir, settings.relative_cwd))
logging.info('Running %s, cwd=%s' % (settings.command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault('RUN_TEST_CASES_LOG_FILE',
os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
try:
with tools.Profiler('RunTest'):
return subprocess.call(settings.command, cwd=cwd, env=env)
except OSError:
print >> sys.stderr, 'Failed to run %s; cwd=%s' % (settings.command, cwd)
return 1
finally:
if outdir:
rmtree(outdir)
def run_tha_test(isolated_hash, storage, cache, leak_temp_dir, extra_args):
"""Downloads the dependencies in the cache, hardlinks them into a temporary
directory and runs the executable from there.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
isolated_hash: the SHA-1 of the .isolated file that must be retrieved to
recreate the tree of files to run the target executable.
storage: an isolateserver.Storage object to retrieve remote objects. This
object has a reference to an isolateserver.StorageApi, which does
the actual I/O.
cache: an isolateserver.LocalCache to keep from retrieving the same objects
constantly by caching the objects retrieved. Can be on-disk or
in-memory.
leak_temp_dir: if true, the temporary directory will be deliberately leaked
for later examination.
extra_args: optional arguments to add to the command stated in the .isolate
file.
"""
run_dir = make_temp_dir(u'run_tha_test', cache.cache_dir)
out_dir = unicode(make_temp_dir(u'isolated_out', cache.cache_dir))
result = 0
try:
try:
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
except isolated_format.IsolatedError:
on_error.report(None)
return 1
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
command = process_command(command, out_dir)
logging.info('Running %s, cwd=%s' % (command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault('RUN_TEST_CASES_LOG_FILE',
os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
sys.stdout.flush()
with tools.Profiler('RunTest'):
try:
with subprocess42.Popen_with_handler(command, cwd=cwd, env=env) as p:
p.communicate()
result = p.returncode
except OSError:
on_error.report('Failed to run %s; cwd=%s' % (command, cwd))
result = 1
logging.info(
'Command finished with exit code %d (%s)',
result, hex(0xffffffff & result))
finally:
try:
if leak_temp_dir:
logging.warning('Deliberately leaking %s for later examination',
run_dir)
else:
try:
if not file_path.rmtree(run_dir):
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
result = result or 1
except OSError:
logging.warning('Leaking %s', run_dir)
result = 1
# HACK(vadimsh): On Windows rmtree(run_dir) call above has
# a synchronization effect: it finishes only when all task child processes
# terminate (since a running process locks *.exe file). Examine out_dir
# only after that call completes (since child processes may
# write to out_dir too and we need to wait for them to finish).
# Upload out_dir and generate a .isolated file out of this directory.
# It is only done if files were written in the directory.
if os.path.isdir(out_dir) and os.listdir(out_dir):
with tools.Profiler('ArchiveOutput'):
results = isolateserver.archive_files_to_storage(
storage, [out_dir], None)
# TODO(maruel): Implement side-channel to publish this information.
output_data = {
'hash': results[0][0],
'namespace': storage.namespace,
'storage': storage.location,
}
sys.stdout.flush()
print(
'[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
tools.format_json(output_data, dense=True))
finally:
try:
if os.path.isdir(out_dir) and not file_path.rmtree(out_dir):
result = result or 1
except OSError:
# The error was already printed out. Report it but that's it. Only
# report on non-Windows or on Windows when the process had succeeded.
# Due to the way file sharing works on Windows, it's sadly expected that
# file deletion may fail when a test failed.
if sys.platform != 'win32' or not result:
on_error.report(None)
result = 1
return result
def CMDreproduce(parser, args):
"""Runs a task locally that was triggered on the server.
This running locally the same commands that have been run on the bot. The data
downloaded will be in a subdirectory named 'work' of the current working
directory.
You can pass further additional arguments to the target command by passing
them after --.
"""
parser.add_option(
'--output-dir', metavar='DIR', default='',
help='Directory that will have results stored into')
options, args = parser.parse_args(args)
extra_args = []
if not args:
parser.error('Must specify exactly one task id.')
if len(args) > 1:
if args[1] == '--':
if len(args) > 2:
extra_args = args[2:]
else:
extra_args = args[1:]
url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0]
request = net.url_read_json(url)
if not request:
print >> sys.stderr, 'Failed to retrieve request data for the task'
return 1
workdir = unicode(os.path.abspath('work'))
if fs.isdir(workdir):
parser.error('Please delete the directory \'work\' first')
fs.mkdir(workdir)
properties = request['properties']
env = None
if properties.get('env'):
env = os.environ.copy()
logging.info('env: %r', properties['env'])
for i in properties['env']:
key = i['key'].encode('utf-8')
if not i['value']:
env.pop(key, None)
else:
env[key] = i['value'].encode('utf-8')
if properties.get('inputs_ref'):
# Create the tree.
with isolateserver.get_storage(
properties['inputs_ref']['isolatedserver'],
properties['inputs_ref']['namespace']) as storage:
bundle = isolateserver.fetch_isolated(
properties['inputs_ref']['isolated'],
storage,
isolateserver.MemoryCache(file_mode_mask=0700),
workdir)
command = bundle.command
if bundle.relative_cwd:
workdir = os.path.join(workdir, bundle.relative_cwd)
command.extend(properties.get('extra_args') or [])
# https://github.com/luci/luci-py/blob/master/appengine/swarming/doc/Magic-Values.md
new_command = run_isolated.process_command(command, options.output_dir)
if not options.output_dir and new_command != command:
parser.error('The task has outputs, you must use --output-dir')
command = new_command
else:
command = properties['command']
try:
return subprocess.call(command + extra_args, env=env, cwd=workdir)
except OSError as e:
print >> sys.stderr, 'Failed to run: %s' % ' '.join(command)
print >> sys.stderr, str(e)
return 1
def run_tha_test(isolated_hash, storage, cache, extra_args):
"""Downloads the dependencies in the cache, hardlinks them into a temporary
directory and runs the executable from there.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
isolated_hash: the sha-1 of the .isolated file that must be retrieved to
recreate the tree of files to run the target executable.
storage: an isolateserver.Storage object to retrieve remote objects. This
object has a reference to an isolateserver.StorageApi, which does
the actual I/O.
cache: an isolateserver.LocalCache to keep from retrieving the same objects
constantly by caching the objects retrieved. Can be on-disk or
in-memory.
extra_args: optional arguments to add to the command stated in the .isolate
file.
"""
run_dir = make_temp_dir('run_tha_test', cache.cache_dir)
out_dir = unicode(make_temp_dir('isolated_out', cache.cache_dir))
result = 0
try:
try:
settings = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
except isolateserver.ConfigError:
on_error.report(None)
return 1
change_tree_read_only(run_dir, settings.read_only)
cwd = os.path.normpath(os.path.join(run_dir, settings.relative_cwd))
command = settings.command + extra_args
# subprocess.call doesn't consider 'cwd' when searching for executable.
# Yet isolate can specify command relative to 'cwd'. Convert it to absolute
# path if necessary.
if not os.path.isabs(command[0]):
command[0] = os.path.abspath(os.path.join(cwd, command[0]))
command = process_command(command, out_dir)
logging.info('Running %s, cwd=%s' % (command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault('RUN_TEST_CASES_LOG_FILE',
os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
try:
sys.stdout.flush()
with tools.Profiler('RunTest'):
result = subprocess.call(command, cwd=cwd, env=env)
logging.info(
'Command finished with exit code %d (%s)',
result, hex(0xffffffff & result))
except OSError:
on_error.report('Failed to run %s; cwd=%s' % (command, cwd))
result = 1
finally:
try:
try:
if not rmtree(run_dir):
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing the\n'
'task because of it. No zombie process can outlive a successful\n'
'task run and still be marked as successful. Fix your stuff.')
result = result or 1
except OSError:
logging.warning('Leaking %s', run_dir)
result = 1
# HACK(vadimsh): On Windows rmtree(run_dir) call above has
# a synchronization effect: it finishes only when all task child processes
# terminate (since a running process locks *.exe file). Examine out_dir
# only after that call completes (since child processes may
# write to out_dir too and we need to wait for them to finish).
# Upload out_dir and generate a .isolated file out of this directory.
# It is only done if files were written in the directory.
if os.listdir(out_dir):
with tools.Profiler('ArchiveOutput'):
results = isolateserver.archive_files_to_storage(
storage, [out_dir], None)
# TODO(maruel): Implement side-channel to publish this information.
output_data = {
'hash': results[0][0],
'namespace': storage.namespace,
'storage': storage.location,
}
sys.stdout.flush()
print(
'[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
tools.format_json(output_data, dense=True))
finally:
try:
if os.path.isdir(out_dir) and not rmtree(out_dir):
result = result or 1
except OSError:
# The error was already printed out. Report it but that's it.
on_error.report(None)
result = 1
return result
def CMDreproduce(parser, args):
"""Runs a task locally that was triggered on the server.
This running locally the same commands that have been run on the bot. The data
downloaded will be in a subdirectory named 'work' of the current working
directory.
You can pass further additional arguments to the target command by passing
them after --.
"""
options, args = parser.parse_args(args)
extra_args = []
if not args:
parser.error('Must specify exactly one task id.')
if len(args) > 1:
if args[1] == '--':
if len(args) > 2:
extra_args = args[2:]
else:
extra_args = args[1:]
url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0]
request = net.url_read_json(url)
if not request:
print >> sys.stderr, 'Failed to retrieve request data for the task'
return 1
workdir = unicode(os.path.abspath('work'))
if not fs.isdir(workdir):
fs.mkdir(workdir)
properties = request['properties']
env = None
if properties.get('env'):
env = os.environ.copy()
logging.info('env: %r', properties['env'])
for i in properties['env']:
key = i['key'].encode('utf-8')
if not i['value']:
env.pop(key, None)
else:
env[key] = i['value'].encode('utf-8')
if properties.get('inputs_ref'):
# Create the tree.
with isolateserver.get_storage(
properties['inputs_ref']['isolatedserver'],
properties['inputs_ref']['namespace']) as storage:
bundle = isolateserver.fetch_isolated(
properties['inputs_ref']['isolated'],
storage,
isolateserver.MemoryCache(file_mode_mask=0700),
workdir,
False)
command = bundle.command
if bundle.relative_cwd:
workdir = os.path.join(workdir, bundle.relative_cwd)
else:
command = properties['command']
try:
return subprocess.call(command + extra_args, env=env, cwd=workdir)
except OSError as e:
print >> sys.stderr, 'Failed to run: %s' % ' '.join(command)
print >> sys.stderr, str(e)
return 1
def map_and_run(
isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout,
grace_period, extra_args):
"""Maps and run the command. Returns metadata about the result."""
# TODO(maruel): Include performance statistics.
result = {
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': None,
'outputs_ref': None,
'version': 2,
}
if root_dir:
if not fs.isdir(root_dir):
fs.makedirs(root_dir, 0700)
prefix = u''
else:
root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
prefix = u'isolated_'
run_dir = make_temp_dir(prefix + u'run', root_dir)
out_dir = make_temp_dir(prefix + u'out', root_dir)
tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
try:
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
result['exit_code'], result['had_hard_timeout'] = run_command(
process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
grace_period)
except Exception as e:
# An internal error occured. Report accordingly so the swarming task will be
# retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
finally:
try:
if leak_temp_dir:
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the run directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
result['outputs_ref'], success = delete_and_upload(
storage, out_dir, leak_temp_dir)
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
def run_tha_test(isolated_hash, storage, cache, extra_args):
"""Downloads the dependencies in the cache, hardlinks them into a temporary
directory and runs the executable from there.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
isolated_hash: the sha-1 of the .isolated file that must be retrieved to
recreate the tree of files to run the target executable.
storage: an isolateserver.Storage object to retrieve remote objects. This
object has a reference to an isolateserver.StorageApi, which does
the actual I/O.
cache: an isolateserver.LocalCache to keep from retrieving the same objects
constantly by caching the objects retrieved. Can be on-disk or
in-memory.
extra_args: optional arguments to add to the command stated in the .isolate
file.
"""
run_dir = make_temp_dir('run_tha_test', cache.cache_dir)
out_dir = unicode(tempfile.mkdtemp(prefix='run_tha_test'))
result = 0
try:
try:
settings = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
except isolateserver.ConfigError as e:
tools.report_error(e)
result = 1
return result
change_tree_read_only(run_dir, settings.read_only)
cwd = os.path.normpath(os.path.join(run_dir, settings.relative_cwd))
command = settings.command + extra_args
# subprocess.call doesn't consider 'cwd' when searching for executable.
# Yet isolate can specify command relative to 'cwd'. Convert it to absolute
# path if necessary.
if not os.path.isabs(command[0]):
command[0] = os.path.abspath(os.path.join(cwd, command[0]))
command = process_command(command, out_dir)
logging.info('Running %s, cwd=%s' % (command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault('RUN_TEST_CASES_LOG_FILE',
os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
try:
with tools.Profiler('RunTest'):
result = subprocess.call(command, cwd=cwd, env=env)
except OSError as e:
tools.report_error('Failed to run %s; cwd=%s: %s' % (command, cwd, e))
result = 1
# Upload out_dir and generate a .isolated file out of this directory. It is
# only done if files were written in the directory.
if os.listdir(out_dir):
with tools.Profiler('ArchiveOutput'):
results = isolateserver.archive_files_to_storage(
storage, [out_dir], None)
# TODO(maruel): Implement side-channel to publish this information.
output_data = {
'hash': results[0][0],
'namespace': storage.namespace,
'storage': storage.location,
}
sys.stdout.flush()
sys.stderr.flush()
print(
'[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
tools.format_json(output_data, dense=True))
finally:
try:
rmtree(out_dir)
finally:
try:
rmtree(run_dir)
except OSError:
logging.warning('Leaking %s', run_dir)
# Swallow the exception so it doesn't generate an infrastructure error.
#
# It usually happens on Windows when a child process is not properly
# terminated, usually because of a test case starting child processes
# that time out. This causes files to be locked and it becomes
# impossible to delete them.
#
# Only report an infrastructure error if the test didn't fail. This is
# because a swarming bot will likely not reboot. This situation will
# cause accumulation of temporary hardlink trees.
if not result:
raise
return result
def map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir,
extra_args):
"""Maps and run the command. Returns metadata about the result."""
# TODO(maruel): Include performance statistics.
result = {
'exit_code': None,
'internal_failure': None,
'outputs_ref': None,
'version': 1,
}
if root_dir:
if not os.path.isdir(root_dir):
os.makedirs(root_dir, 0700)
prefix = u''
else:
root_dir = os.path.dirname(
cache.cache_dir) if cache.cache_dir else None
prefix = u'isolated_'
run_dir = make_temp_dir(prefix + u'run', root_dir)
out_dir = make_temp_dir(prefix + u'out', root_dir)
tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
try:
bundle = isolateserver.fetch_isolated(isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
result['exit_code'] = run_command(process_command(command, out_dir),
cwd, tmp_dir)
except Exception as e:
# An internal error occured. Report accordingly so the swarming task will be
# retried automatically.
logging.error('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
finally:
try:
if leak_temp_dir:
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
if os.path.isdir(run_dir) and not file_path.rmtree(run_dir):
# On Windows rmtree(run_dir) call above has a synchronization effect:
# it finishes only when all task child processes terminate (since a
# running process locks *.exe file). Examine out_dir only after that
# call completes (since child processes may write to out_dir too and
# we need to wait for them to finish).
print >> sys.stderr, (
'Failed to delete the run directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
if os.path.isdir(tmp_dir) and not file_path.rmtree(tmp_dir):
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
result['outputs_ref'], success = delete_and_upload(
storage, out_dir, leak_temp_dir)
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
logging.error('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
def map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir,
hard_timeout, grace_period, extra_args):
"""Maps and run the command. Returns metadata about the result."""
# TODO(maruel): Include performance statistics.
result = {
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': None,
'outputs_ref': None,
'version': 2,
}
if root_dir:
if not fs.isdir(root_dir):
fs.makedirs(root_dir, 0700)
prefix = u''
else:
root_dir = os.path.dirname(
cache.cache_dir) if cache.cache_dir else None
prefix = u'isolated_'
run_dir = make_temp_dir(prefix + u'run', root_dir)
out_dir = make_temp_dir(prefix + u'out', root_dir)
tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
try:
try:
bundle = isolateserver.fetch_isolated(isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
except isolateserver.IsolatedErrorNoCommand:
# Handle this as a task failure, not an internal failure.
sys.stderr.write(
'<The .isolated doesn\'t declare any command to run!>\n'
'<Check your .isolate for missing \'command\' variable>\n')
if os.environ.get('SWARMING_TASK_ID'):
# Give an additional hint when running as a swarming task.
sys.stderr.write('<This occurs at the \'isolate\' step>\n')
result['exit_code'] = 1
return result
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
result['exit_code'], result['had_hard_timeout'] = run_command(
process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
grace_period)
except Exception as e:
# An internal error occured. Report accordingly so the swarming task will be
# retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
finally:
try:
if leak_temp_dir:
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the run directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
result['outputs_ref'], success = delete_and_upload(
storage, out_dir, leak_temp_dir)
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
def run_tha_test(isolated_hash, storage, cache, extra_args):
"""Downloads the dependencies in the cache, hardlinks them into a temporary
directory and runs the executable from there.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
isolated_hash: the sha-1 of the .isolated file that must be retrieved to
recreate the tree of files to run the target executable.
storage: an isolateserver.Storage object to retrieve remote objects. This
object has a reference to an isolateserver.StorageApi, which does
the actual I/O.
cache: an isolateserver.LocalCache to keep from retrieving the same objects
constantly by caching the objects retrieved. Can be on-disk or
in-memory.
extra_args: optional arguments to add to the command stated in the .isolate
file.
"""
run_dir = make_temp_dir("run_tha_test", cache.cache_dir)
out_dir = unicode(make_temp_dir("isolated_out", cache.cache_dir))
result = 0
try:
try:
settings = isolateserver.fetch_isolated(
isolated_hash=isolated_hash, storage=storage, cache=cache, outdir=run_dir, require_command=True
)
except isolateserver.ConfigError as e:
tools.report_error(e)
return 1
change_tree_read_only(run_dir, settings.read_only)
cwd = os.path.normpath(os.path.join(run_dir, settings.relative_cwd))
command = settings.command + extra_args
# subprocess.call doesn't consider 'cwd' when searching for executable.
# Yet isolate can specify command relative to 'cwd'. Convert it to absolute
# path if necessary.
if not os.path.isabs(command[0]):
command[0] = os.path.abspath(os.path.join(cwd, command[0]))
command = process_command(command, out_dir)
logging.info("Running %s, cwd=%s" % (command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault("RUN_TEST_CASES_LOG_FILE", os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
try:
sys.stdout.flush()
with tools.Profiler("RunTest"):
result = subprocess.call(command, cwd=cwd, env=env)
logging.info("Command finished with exit code %d (%s)", result, hex(0xFFFFFFFF & result))
except OSError as e:
tools.report_error("Failed to run %s; cwd=%s: %s" % (command, cwd, e))
result = 1
finally:
try:
try:
rmtree(run_dir)
except OSError:
logging.warning("Leaking %s", run_dir)
# Swallow the exception so it doesn't generate an infrastructure error.
#
# It usually happens on Windows when a child process is not properly
# terminated, usually because of a test case starting child processes
# that time out. This causes files to be locked and it becomes
# impossible to delete them.
#
# Only report an infrastructure error if the test didn't fail. This is
# because a swarming bot will likely not reboot. This situation will
# cause accumulation of temporary hardlink trees.
if not result:
raise
# HACK(vadimsh): On Windows rmtree(run_dir) call above has
# a synchronization effect: it finishes only when all task child processes
# terminate (since a running process locks *.exe file). Examine out_dir
# only after that call completes (since child processes may
# write to out_dir too and we need to wait for them to finish).
# Upload out_dir and generate a .isolated file out of this directory.
# It is only done if files were written in the directory.
if os.listdir(out_dir):
with tools.Profiler("ArchiveOutput"):
results = isolateserver.archive_files_to_storage(storage, [out_dir], None)
# TODO(maruel): Implement side-channel to publish this information.
output_data = {"hash": results[0][0], "namespace": storage.namespace, "storage": storage.location}
sys.stdout.flush()
print("[run_isolated_out_hack]%s[/run_isolated_out_hack]" % tools.format_json(output_data, dense=True))
finally:
rmtree(out_dir)
return result
def run_tha_test(isolated_hash, storage, cache, leak_temp_dir, extra_args):
"""Downloads the dependencies in the cache, hardlinks them into a temporary
directory and runs the executable from there.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
isolated_hash: the SHA-1 of the .isolated file that must be retrieved to
recreate the tree of files to run the target executable.
storage: an isolateserver.Storage object to retrieve remote objects. This
object has a reference to an isolateserver.StorageApi, which does
the actual I/O.
cache: an isolateserver.LocalCache to keep from retrieving the same objects
constantly by caching the objects retrieved. Can be on-disk or
in-memory.
leak_temp_dir: if true, the temporary directory will be deliberately leaked
for later examination.
extra_args: optional arguments to add to the command stated in the .isolate
file.
"""
tmp_root = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
run_dir = make_temp_dir(u'run_tha_test', tmp_root)
out_dir = unicode(make_temp_dir(u'isolated_out', tmp_root))
result = 0
try:
try:
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir,
require_command=True)
except isolated_format.IsolatedError:
on_error.report(None)
return 1
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
command = process_command(command, out_dir)
logging.info('Running %s, cwd=%s' % (command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault('RUN_TEST_CASES_LOG_FILE',
os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
sys.stdout.flush()
with tools.Profiler('RunTest'):
try:
with subprocess42.Popen_with_handler(command, cwd=cwd, env=env) as p:
p.communicate()
result = p.returncode
except OSError:
on_error.report('Failed to run %s; cwd=%s' % (command, cwd))
result = 1
logging.info(
'Command finished with exit code %d (%s)',
result, hex(0xffffffff & result))
finally:
try:
if leak_temp_dir:
logging.warning('Deliberately leaking %s for later examination',
run_dir)
else:
try:
if not file_path.rmtree(run_dir):
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
result = result or 1
except OSError as exc:
logging.error('Leaking run_dir %s: %s', run_dir, exc)
result = 1
# HACK(vadimsh): On Windows rmtree(run_dir) call above has
# a synchronization effect: it finishes only when all task child processes
# terminate (since a running process locks *.exe file). Examine out_dir
# only after that call completes (since child processes may
# write to out_dir too and we need to wait for them to finish).
# Upload out_dir and generate a .isolated file out of this directory.
# It is only done if files were written in the directory.
if os.path.isdir(out_dir) and os.listdir(out_dir):
with tools.Profiler('ArchiveOutput'):
try:
results = isolateserver.archive_files_to_storage(
storage, [out_dir], None)
except isolateserver.Aborted:
# This happens when a signal SIGTERM was received while uploading
# data. There is 2 causes:
# - The task was too slow and was about to be killed anyway due to
# exceeding the hard timeout.
# - The amount of data uploaded back is very large and took too much
# time to archive.
#
# There's 3 options to handle this:
# - Ignore the upload failure as a silent failure. This can be
# detected client side by the fact no result file exists.
# - Return as if the task failed. This is not factually correct.
# - Return an internal failure. Sadly, it's impossible at this level
# at the moment.
#
# For now, silently drop the upload.
#
# In any case, the process only has a very short grace period so it
# needs to exit right away.
sys.stderr.write('Received SIGTERM while uploading')
results = None
if results:
# TODO(maruel): Implement side-channel to publish this information.
output_data = {
'hash': results[0][0],
'namespace': storage.namespace,
'storage': storage.location,
}
sys.stdout.flush()
print(
'[run_isolated_out_hack]%s[/run_isolated_out_hack]' %
tools.format_json(output_data, dense=True))
finally:
try:
if os.path.isdir(out_dir) and not file_path.rmtree(out_dir):
logging.error('Had difficulties removing out_dir %s', out_dir)
result = result or 1
except OSError as exc:
# Only report on non-Windows or on Windows when the process had
# succeeded. Due to the way file sharing works on Windows, it's sadly
# expected that file deletion may fail when a test failed.
logging.error('Failed to remove out_dir %s: %s', out_dir, exc)
if sys.platform != 'win32' or not result:
on_error.report(None)
result = 1
return result
def map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir, extra_args):
"""Maps and run the command. Returns metadata about the result."""
# TODO(maruel): Include performance statistics.
result = {"exit_code": None, "internal_failure": None, "outputs_ref": None, "version": 1}
if root_dir:
if not os.path.isdir(root_dir):
os.makedirs(root_dir, 0700)
prefix = u""
else:
root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
prefix = u"isolated_"
run_dir = make_temp_dir(prefix + u"run", root_dir)
out_dir = make_temp_dir(prefix + u"out", root_dir)
tmp_dir = make_temp_dir(prefix + u"tmp", root_dir)
try:
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash, storage=storage, cache=cache, outdir=run_dir, require_command=True
)
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
result["exit_code"] = run_command(process_command(command, out_dir), cwd, tmp_dir)
except Exception as e:
# An internal error occured. Report accordingly so the swarming task will be
# retried automatically.
logging.error("internal failure: %s", e)
result["internal_failure"] = str(e)
on_error.report(None)
finally:
try:
if leak_temp_dir:
logging.warning("Deliberately leaking %s for later examination", run_dir)
else:
if os.path.isdir(run_dir) and not file_path.rmtree(run_dir):
# On Windows rmtree(run_dir) call above has a synchronization effect:
# it finishes only when all task child processes terminate (since a
# running process locks *.exe file). Examine out_dir only after that
# call completes (since child processes may write to out_dir too and
# we need to wait for them to finish).
print >>sys.stderr, (
"Failed to delete the run directory, forcibly failing\n"
"the task because of it. No zombie process can outlive a\n"
"successful task run and still be marked as successful.\n"
"Fix your stuff."
)
if result["exit_code"] == 0:
result["exit_code"] = 1
if os.path.isdir(tmp_dir) and not file_path.rmtree(tmp_dir):
print >>sys.stderr, (
"Failed to delete the temporary directory, forcibly failing\n"
"the task because of it. No zombie process can outlive a\n"
"successful task run and still be marked as successful.\n"
"Fix your stuff."
)
if result["exit_code"] == 0:
result["exit_code"] = 1
# This deletes out_dir if leak_temp_dir is not set.
result["outputs_ref"], success = delete_and_upload(storage, out_dir, leak_temp_dir)
if not success and result["exit_code"] == 0:
result["exit_code"] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
logging.error("Leaking out_dir %s: %s", out_dir, e)
result["internal_failure"] = str(e)
return result
def run_tha_test(isolated_hash, storage, cache, algo, extra_args):
"""Downloads the dependencies in the cache, hardlinks them into a temporary
directory and runs the executable from there.
A temporary directory is created to hold the output files. The content inside
this directory will be uploaded back to |storage| packaged as a .isolated
file.
Arguments:
isolated_hash: the sha-1 of the .isolated file that must be retrieved to
recreate the tree of files to run the target executable.
storage: an isolateserver.Storage object to retrieve remote objects. This
object has a reference to an isolateserver.StorageApi, which does
the actual I/O.
cache: an isolateserver.LocalCache to keep from retrieving the same objects
constantly by caching the objects retrieved. Can be on-disk or
in-memory.
algo: an hashlib class to hash content. Usually hashlib.sha1.
extra_args: optional arguments to add to the command stated in the .isolate
file.
"""
run_dir = make_temp_dir('run_tha_test', cache.cache_dir)
out_dir = unicode(tempfile.mkdtemp(prefix='run_tha_test'))
result = 0
try:
try:
settings = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
algo=algo,
outdir=run_dir,
os_flavor=get_flavor(),
require_command=True)
except isolateserver.ConfigError as e:
tools.report_error(e)
result = 1
return result
change_tree_read_only(run_dir, settings.read_only)
cwd = os.path.normpath(os.path.join(run_dir, settings.relative_cwd))
command = settings.command + extra_args
# subprocess.call doesn't consider 'cwd' when searching for executable.
# Yet isolate can specify command relative to 'cwd'. Convert it to absolute
# path if necessary.
if not os.path.isabs(command[0]):
command[0] = os.path.abspath(os.path.join(cwd, command[0]))
command = process_command(command, out_dir)
logging.info('Running %s, cwd=%s' % (command, cwd))
# TODO(csharp): This should be specified somewhere else.
# TODO(vadimsh): Pass it via 'env_vars' in manifest.
# Add a rotating log file if one doesn't already exist.
env = os.environ.copy()
if MAIN_DIR:
env.setdefault('RUN_TEST_CASES_LOG_FILE',
os.path.join(MAIN_DIR, RUN_TEST_CASES_LOG))
try:
with tools.Profiler('RunTest'):
result = subprocess.call(command, cwd=cwd, env=env)
except OSError as e:
tools.report_error('Failed to run %s; cwd=%s: %s' % (command, cwd, e))
result = 1
# Upload out_dir and generate a .isolated file out of this directory. It is
# only done if files were written in the directory.
if os.listdir(out_dir):
with tools.Profiler('ArchiveOutput'):
results = isolateserver.archive_files_to_storage(
storage, algo, [out_dir], None)
# TODO(maruel): Implement side-channel to publish this information.
print('run_isolated output: %s' % results[0][0])
finally:
try:
rmtree(out_dir)
finally:
try:
rmtree(run_dir)
except OSError:
logging.warning('Leaking %s', run_dir)
# Swallow the exception so it doesn't generate an infrastructure error.
#
# It usually happens on Windows when a child process is not properly
# terminated, usually because of a test case starting child processes
# that time out. This causes files to be locked and it becomes
# impossible to delete them.
#
# Only report an infrastructure error if the test didn't fail. This is
# because a swarming bot will likely not reboot. This situation will
# cause accumulation of temporary hardlink trees.
if not result:
raise
return result
def map_and_run(
isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout,
grace_period, extra_args):
"""Maps and run the command. Returns metadata about the result."""
result = {
'duration': None,
'exit_code': None,
'had_hard_timeout': False,
'internal_failure': None,
'stats': {
# 'download': {
# 'duration': 0.,
# 'initial_number_items': 0,
# 'initial_size': 0,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
# 'upload': {
# 'duration': 0.,
# 'items_cold': '<large.pack()>',
# 'items_hot': '<large.pack()>',
# },
},
'outputs_ref': None,
'version': 3,
}
if root_dir:
if not fs.isdir(root_dir):
fs.makedirs(root_dir, 0700)
prefix = u''
else:
root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None
prefix = u'isolated_'
run_dir = make_temp_dir(prefix + u'run', root_dir)
out_dir = make_temp_dir(prefix + u'out', root_dir)
tmp_dir = make_temp_dir(prefix + u'tmp', root_dir)
try:
start = time.time()
bundle = isolateserver.fetch_isolated(
isolated_hash=isolated_hash,
storage=storage,
cache=cache,
outdir=run_dir)
if not bundle.command:
# Handle this as a task failure, not an internal failure.
sys.stderr.write(
'<The .isolated doesn\'t declare any command to run!>\n'
'<Check your .isolate for missing \'command\' variable>\n')
if os.environ.get('SWARMING_TASK_ID'):
# Give an additional hint when running as a swarming task.
sys.stderr.write('<This occurs at the \'isolate\' step>\n')
result['exit_code'] = 1
return result
result['stats']['download'] = {
'duration': time.time() - start,
'initial_number_items': cache.initial_number_items,
'initial_size': cache.initial_size,
'items_cold': base64.b64encode(large.pack(sorted(cache.added))),
'items_hot': base64.b64encode(
large.pack(sorted(set(cache.linked) - set(cache.added)))),
}
change_tree_read_only(run_dir, bundle.read_only)
cwd = os.path.normpath(os.path.join(run_dir, bundle.relative_cwd))
command = bundle.command + extra_args
file_path.ensure_command_has_abs_path(command, cwd)
sys.stdout.flush()
start = time.time()
try:
result['exit_code'], result['had_hard_timeout'] = run_command(
process_command(command, out_dir), cwd, tmp_dir, hard_timeout,
grace_period)
finally:
result['duration'] = max(time.time() - start, 0)
except Exception as e:
# An internal error occured. Report accordingly so the swarming task will be
# retried automatically.
logging.exception('internal failure: %s', e)
result['internal_failure'] = str(e)
on_error.report(None)
finally:
try:
if leak_temp_dir:
logging.warning(
'Deliberately leaking %s for later examination', run_dir)
else:
# On Windows rmtree(run_dir) call above has a synchronization effect: it
# finishes only when all task child processes terminate (since a running
# process locks *.exe file). Examine out_dir only after that call
# completes (since child processes may write to out_dir too and we need
# to wait for them to finish).
if fs.isdir(run_dir):
try:
success = file_path.rmtree(run_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the run directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
if fs.isdir(tmp_dir):
try:
success = file_path.rmtree(tmp_dir)
except OSError as e:
logging.error('Failure with %s', e)
success = False
if not success:
print >> sys.stderr, (
'Failed to delete the temporary directory, forcibly failing\n'
'the task because of it. No zombie process can outlive a\n'
'successful task run and still be marked as successful.\n'
'Fix your stuff.')
if result['exit_code'] == 0:
result['exit_code'] = 1
# This deletes out_dir if leak_temp_dir is not set.
start = time.time()
result['outputs_ref'], success, cold, hot = delete_and_upload(
storage, out_dir, leak_temp_dir)
result['stats']['upload'] = {
'duration': time.time() - start,
'items_cold': base64.b64encode(large.pack(cold)),
'items_hot': base64.b64encode(large.pack(hot)),
}
if not success and result['exit_code'] == 0:
result['exit_code'] = 1
except Exception as e:
# Swallow any exception in the main finally clause.
logging.exception('Leaking out_dir %s: %s', out_dir, e)
result['internal_failure'] = str(e)
return result
