def read(complete_state):
"""Reads a trace and returns the .isolate dictionary."""
api = trace_inputs.get_api()
logfile = complete_state.result_file + '.log'
if not os.path.isfile(logfile):
raise ExecutionError(
'No log file \'%s\' to read, did you forget to \'trace\'?' % logfile)
try:
results = trace_inputs.load_trace(
logfile, complete_state.root_dir, api, isolate_common.default_blacklist)
tracked, touched = isolate_common.split_touched(results.existent)
value = isolate_common.generate_isolate(
tracked,
[],
touched,
complete_state.root_dir,
complete_state.saved_state.variables,
complete_state.result.relative_cwd)
return value
except trace_inputs.TracingFailure, e:
raise ExecutionError(
'Reading traces failed for: %s\n%s' %
(' '.join(complete_state.result.command), str(e)))
def trace_test_cases(
executable, root_dir, cwd_dir, variables, test_cases, jobs, output_file):
"""Traces test cases one by one."""
assert not os.path.isabs(cwd_dir)
assert os.path.isabs(root_dir) and os.path.isdir(root_dir)
assert os.path.isfile(executable) and os.path.isabs(executable)
if not test_cases:
return 0
# Resolve any symlink.
root_dir = os.path.realpath(root_dir)
full_cwd_dir = os.path.normpath(os.path.join(root_dir, cwd_dir))
assert os.path.isdir(full_cwd_dir)
logname = output_file + '.logs'
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
api = trace_inputs.get_api()
api.clean_trace(logname)
with api.get_tracer(logname) as tracer:
function = Tracer(tracer, executable, full_cwd_dir, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
values = pool.join(progress, 0.1)
print ''
print '%.1fs Done post-processing logs. Parsing logs.' % (
time.time() - progress.start)
results = api.parse_log(logname, isolate_common.default_blacklist)
print '%.1fs Done parsing logs.' % (
time.time() - progress.start)
# Strips to root_dir.
results_processed = {}
for item in results:
if 'results' in item:
item = item.copy()
item['results'] = item['results'].strip_root(root_dir)
results_processed[item['trace']] = item
else:
print >> sys.stderr, 'Got exception while tracing %s: %s' % (
item['trace'], item['exception'])
print '%.1fs Done stripping root.' % (
time.time() - progress.start)
# Flatten.
flattened = {}
for item_list in values:
for item in item_list:
if item['valid']:
test_case = item['test_case']
tracename = test_case.replace('/', '-')
flattened[test_case] = results_processed[tracename].copy()
item_results = flattened[test_case]['results']
tracked, touched = isolate_common.split_touched(item_results.existent)
flattened[test_case].update({
'processes': len(list(item_results.process.all)),
'results': item_results.flatten(),
'duration': item['duration'],
'returncode': item['returncode'],
'valid': item['valid'],
'variables':
isolate_common.generate_simplified(
tracked,
[],
touched,
root_dir,
variables,
cwd_dir),
})
del flattened[test_case]['trace']
print '%.1fs Done flattening.' % (
time.time() - progress.start)
# Make it dense if there is more than 20 results.
trace_inputs.write_json(
output_file,
flattened,
False)
# Also write the .isolate file.
# First, get all the files from all results. Use a map to remove dupes.
files = {}
for item in results_processed.itervalues():
files.update((f.full_path, f) for f in item['results'].existent)
# Convert back to a list, discard the keys.
files = files.values()
tracked, touched = isolate_common.split_touched(files)
value = isolate_common.generate_isolate(
tracked,
[],
touched,
root_dir,
variables,
cwd_dir)
with open('%s.isolate' % output_file, 'wb') as f:
isolate_common.pretty_print(value, f)
return 0
def trace_test_cases(executable, root_dir, cwd_dir, variables, test_cases,
jobs, output_file):
"""Traces test cases one by one."""
assert not os.path.isabs(cwd_dir)
assert os.path.isabs(root_dir) and os.path.isdir(root_dir)
assert os.path.isfile(executable) and os.path.isabs(executable)
if not test_cases:
return 0
# Resolve any symlink.
root_dir = os.path.realpath(root_dir)
full_cwd_dir = os.path.normpath(os.path.join(root_dir, cwd_dir))
assert os.path.isdir(full_cwd_dir)
logname = output_file + '.logs'
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
api = trace_inputs.get_api()
api.clean_trace(logname)
with api.get_tracer(logname) as tracer:
function = Tracer(tracer, executable, full_cwd_dir, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
values = pool.join(progress, 0.1)
print ''
print '%.1fs Done post-processing logs. Parsing logs.' % (time.time() -
progress.start)
results = api.parse_log(logname, isolate_common.default_blacklist)
print '%.1fs Done parsing logs.' % (time.time() - progress.start)
# Strips to root_dir.
results_processed = {}
for item in results:
if 'results' in item:
item = item.copy()
item['results'] = item['results'].strip_root(root_dir)
results_processed[item['trace']] = item
else:
print >> sys.stderr, 'Got exception while tracing %s: %s' % (
item['trace'], item['exception'])
print '%.1fs Done stripping root.' % (time.time() - progress.start)
# Flatten.
flattened = {}
for item_list in values:
for item in item_list:
if item['valid']:
test_case = item['test_case']
tracename = test_case.replace('/', '-')
flattened[test_case] = results_processed[tracename].copy()
item_results = flattened[test_case]['results']
tracked, touched = isolate_common.split_touched(
item_results.existent)
flattened[test_case].update({
'processes':
len(list(item_results.process.all)),
'results':
item_results.flatten(),
'duration':
item['duration'],
'returncode':
item['returncode'],
'valid':
item['valid'],
'variables':
isolate_common.generate_simplified(tracked, [], touched,
root_dir, variables,
cwd_dir),
})
del flattened[test_case]['trace']
print '%.1fs Done flattening.' % (time.time() - progress.start)
# Make it dense if there is more than 20 results.
trace_inputs.write_json(output_file, flattened, False)
# Also write the .isolate file.
# First, get all the files from all results. Use a map to remove dupes.
files = {}
for item in results_processed.itervalues():
files.update((f.full_path, f) for f in item['results'].existent)
# Convert back to a list, discard the keys.
files = files.values()
tracked, touched = isolate_common.split_touched(files)
value = isolate_common.generate_isolate(tracked, [], touched, root_dir,
variables, cwd_dir)
with open('%s.isolate' % output_file, 'wb') as f:
isolate_common.pretty_print(value, f)
return 0
