def write_details(logname, outfile, root_dir, blacklist, results):
"""Writes an .test_cases file with all the information about each test
case.
"""
api = trace_inputs.get_api()
logs = dict(
(i.pop('trace'), i) for i in api.parse_log(logname, blacklist, None))
results_processed = {}
exception = None
for items in results:
item = items[-1]
assert item['valid']
# Load the results;
log_dict = logs[item['tracename']]
if log_dict.get('exception'):
exception = exception or log_dict['exception']
continue
trace_result = log_dict['results']
if root_dir:
trace_result = trace_result.strip_root(root_dir)
results_processed[item['test_case']] = {
'trace': trace_result.flatten(),
'duration': item['duration'],
'output': item['output'],
'returncode': item['returncode'],
}
# Make it dense if there is more than 20 results.
trace_inputs.write_json(outfile, results_processed,
len(results_processed) > 20)
if exception:
raise exception[0], exception[1], exception[2]
def write_details(logname, outfile, root_dir, blacklist, results):
"""Writes an .test_cases file with all the information about each test
case.
"""
api = trace_inputs.get_api()
logs = dict((i.pop("trace"), i) for i in api.parse_log(logname, blacklist, None))
results_processed = {}
exception = None
for items in results:
item = items[-1]
assert item["valid"]
# Load the results;
log_dict = logs[item["tracename"]]
if log_dict.get("exception"):
exception = exception or log_dict["exception"]
continue
trace_result = log_dict["results"]
if root_dir:
trace_result = trace_result.strip_root(root_dir)
results_processed[item["test_case"]] = {
"trace": trace_result.flatten(),
"duration": item["duration"],
"output": item["output"],
"returncode": item["returncode"],
}
# Make it dense if there is more than 20 results.
trace_inputs.write_json(outfile, results_processed, len(results_processed) > 20)
if exception:
raise exception[0], exception[1], exception[2]
def write_details(logname, outfile, root_dir, blacklist, results):
"""Writes an .test_cases file with all the information about each test
case.
"""
api = trace_inputs.get_api()
logs = dict(
(i.pop('trace'), i) for i in api.parse_log(logname, blacklist, None))
results_processed = {}
exception = None
for items in results:
item = items[-1]
assert item['valid']
# Load the results;
log_dict = logs[item['tracename']]
if log_dict.get('exception'):
exception = exception or log_dict['exception']
continue
trace_result = log_dict['results']
if root_dir:
trace_result = trace_result.strip_root(root_dir)
results_processed[item['test_case']] = {
'trace': trace_result.flatten(),
'duration': item['duration'],
'output': item['output'],
'returncode': item['returncode'],
}
# Make it dense if there is more than 20 results.
trace_inputs.write_json(
outfile,
results_processed,
len(results_processed) > 20)
if exception:
raise exception[0], exception[1], exception[2]
def run_test_cases(
executable, whitelist, blacklist, jobs, timeout, stats_only, no_dump):
"""Traces test cases one by one."""
test_cases = get_test_cases(executable, whitelist, blacklist)
if not test_cases:
return
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
function = Runner(executable, os.getcwd(), timeout, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
results = pool.join(progress, 0.1)
duration = time.time() - progress.start
results = dict((item[0]['test_case'], item) for item in results)
if not no_dump:
trace_inputs.write_json('%s.run_test_cases' % executable, results, False)
sys.stderr.write('\n')
total = len(results)
if not total:
return 1
# Classify the results
success = []
flaky = []
fail = []
nb_runs = 0
for test_case in sorted(results):
items = results[test_case]
nb_runs += len(items)
if not any(not i['returncode'] for i in items):
fail.append(test_case)
elif len(items) > 1 and any(not i['returncode'] for i in items):
flaky.append(test_case)
elif len(items) == 1 and items[0]['returncode'] == 0:
success.append(test_case)
else:
assert False, items
if not stats_only:
for test_case in sorted(fail):
# Failed, print the last one:
items = results[test_case]
print items[-1]['output']
for test_case in sorted(flaky):
items = results[test_case]
print '%s is flaky (tried %d times)' % (test_case, len(items))
print 'Success: %4d %5.2f%%' % (len(success), len(success) * 100. / total)
print 'Flaky: %4d %5.2f%%' % (len(flaky), len(flaky) * 100. / total)
print 'Fail: %4d %5.2f%%' % (len(fail), len(fail) * 100. / total)
print '%.1fs Done running %d tests with %d executions. %.1f test/s' % (
duration,
len(results),
nb_runs,
nb_runs / duration)
return 0
def save_files(self):
"""Saves both self.result and self.saved_state."""
trace_inputs.write_json(self.result_file, self.result.flatten(), False)
total_bytes = sum(
i.get('size', 0) for i in self.result.files.itervalues())
if total_bytes:
logging.debug('Total size: %d bytes' % total_bytes)
trace_inputs.write_json(result_to_state(self.result_file),
self.saved_state.flatten(), False)
def run_test_cases(executable, whitelist, blacklist, jobs, timeout, stats_only,
no_dump):
"""Traces test cases one by one."""
test_cases = get_test_cases(executable, whitelist, blacklist)
if not test_cases:
return
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
function = Runner(executable, os.getcwd(), timeout, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
results = pool.join(progress, 0.1)
duration = time.time() - progress.start
results = dict((item[0]['test_case'], item) for item in results)
if not no_dump:
trace_inputs.write_json('%s.run_test_cases' % executable, results,
False)
sys.stderr.write('\n')
total = len(results)
if not total:
return 1
# Classify the results
success = []
flaky = []
fail = []
nb_runs = 0
for test_case in sorted(results):
items = results[test_case]
nb_runs += len(items)
if not any(not i['returncode'] for i in items):
fail.append(test_case)
elif len(items) > 1 and any(not i['returncode'] for i in items):
flaky.append(test_case)
elif len(items) == 1 and items[0]['returncode'] == 0:
success.append(test_case)
else:
assert False, items
if not stats_only:
for test_case in sorted(fail):
# Failed, print the last one:
items = results[test_case]
print items[-1]['output']
for test_case in sorted(flaky):
items = results[test_case]
print '%s is flaky (tried %d times)' % (test_case, len(items))
print 'Success: %4d %5.2f%%' % (len(success), len(success) * 100. / total)
print 'Flaky: %4d %5.2f%%' % (len(flaky), len(flaky) * 100. / total)
print 'Fail: %4d %5.2f%%' % (len(fail), len(fail) * 100. / total)
print '%.1fs Done running %d tests with %d executions. %.1f test/s' % (
duration, len(results), nb_runs, nb_runs / duration)
return 0
def save_files(self):
"""Saves both self.result and self.saved_state."""
logging.debug('Dumping to %s' % self.result_file)
trace_inputs.write_json(self.result_file, self.result.flatten(), False)
total_bytes = sum(i.get('size', 0) for i in self.result.files.itervalues())
if total_bytes:
logging.debug('Total size: %d bytes' % total_bytes)
saved_state_file = result_to_state(self.result_file)
logging.debug('Dumping to %s' % saved_state_file)
trace_inputs.write_json(saved_state_file, self.saved_state.flatten(), False)
def trace_test_cases(
executable, root_dir, cwd_dir, variables, test_cases, jobs, output_file):
"""Traces test cases one by one."""
assert not os.path.isabs(cwd_dir)
assert os.path.isabs(root_dir) and os.path.isdir(root_dir)
assert os.path.isfile(executable) and os.path.isabs(executable)
if not test_cases:
return 0
# Resolve any symlink.
root_dir = os.path.realpath(root_dir)
full_cwd_dir = os.path.normpath(os.path.join(root_dir, cwd_dir))
assert os.path.isdir(full_cwd_dir)
logname = output_file + '.logs'
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
api = trace_inputs.get_api()
api.clean_trace(logname)
with api.get_tracer(logname) as tracer:
function = Tracer(tracer, executable, full_cwd_dir, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
values = pool.join(progress, 0.1)
print ''
print '%.1fs Done post-processing logs. Parsing logs.' % (
time.time() - progress.start)
results = api.parse_log(logname, isolate_common.default_blacklist)
print '%.1fs Done parsing logs.' % (
time.time() - progress.start)
# Strips to root_dir.
results_processed = {}
for item in results:
if 'results' in item:
item = item.copy()
item['results'] = item['results'].strip_root(root_dir)
results_processed[item['trace']] = item
else:
print >> sys.stderr, 'Got exception while tracing %s: %s' % (
item['trace'], item['exception'])
print '%.1fs Done stripping root.' % (
time.time() - progress.start)
# Flatten.
flattened = {}
for item_list in values:
for item in item_list:
if item['valid']:
test_case = item['test_case']
tracename = test_case.replace('/', '-')
flattened[test_case] = results_processed[tracename].copy()
item_results = flattened[test_case]['results']
tracked, touched = isolate_common.split_touched(item_results.existent)
flattened[test_case].update({
'processes': len(list(item_results.process.all)),
'results': item_results.flatten(),
'duration': item['duration'],
'returncode': item['returncode'],
'valid': item['valid'],
'variables':
isolate_common.generate_simplified(
tracked,
[],
touched,
root_dir,
variables,
cwd_dir),
})
del flattened[test_case]['trace']
print '%.1fs Done flattening.' % (
time.time() - progress.start)
# Make it dense if there is more than 20 results.
trace_inputs.write_json(
output_file,
flattened,
False)
# Also write the .isolate file.
# First, get all the files from all results. Use a map to remove dupes.
files = {}
for item in results_processed.itervalues():
files.update((f.full_path, f) for f in item['results'].existent)
# Convert back to a list, discard the keys.
files = files.values()
tracked, touched = isolate_common.split_touched(files)
value = isolate_common.generate_isolate(
tracked,
[],
touched,
root_dir,
variables,
cwd_dir)
with open('%s.isolate' % output_file, 'wb') as f:
isolate_common.pretty_print(value, f)
return 0
def trace_test_cases(executable, root_dir, cwd_dir, variables, whitelist,
blacklist, jobs, index, shards, output_file):
"""Traces test cases one by one."""
test_cases = get_test_cases(executable, whitelist, blacklist, index,
shards)
if not test_cases:
return
# Resolve any symlink.
root_dir = os.path.realpath(root_dir)
full_cwd_dir = os.path.join(root_dir, cwd_dir)
logname = output_file + '.logs'
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
api = trace_inputs.get_api()
api.clean_trace(logname)
with api.get_tracer(logname) as tracer:
function = Tracer(tracer, executable, full_cwd_dir, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
values = pool.join(progress, 0.1)
print ''
print '%.1fs Done post-processing logs. Parsing logs.' % (time.time() -
progress.start)
results = api.parse_log(logname, isolate_common.default_blacklist)
print '%.1fs Done parsing logs.' % (time.time() - progress.start)
# Strips to root_dir.
results_processed = {}
for item in results:
if 'results' in item:
item = item.copy()
item['results'] = item['results'].strip_root(root_dir)
results_processed[item['trace']] = item
else:
print >> sys.stderr, 'Got exception while tracing %s: %s' % (
item['trace'], item['exception'])
print '%.1fs Done stripping root.' % (time.time() - progress.start)
# Flatten.
flattened = {}
for item_list in values:
for item in item_list:
if item['valid']:
test_case = item['test_case']
tracename = test_case.replace('/', '-')
flattened[test_case] = results_processed[tracename].copy()
item_results = flattened[test_case]['results']
flattened[test_case].update({
'processes':
len(list(item_results.process.all)),
'results':
item_results.flatten(),
'duration':
item['duration'],
'returncode':
item['returncode'],
'valid':
item['valid'],
'variables':
isolate_common.generate_simplified(item_results.existent,
root_dir, variables,
cwd_dir),
})
del flattened[test_case]['trace']
print '%.1fs Done flattening.' % (time.time() - progress.start)
# Make it dense if there is more than 20 results.
trace_inputs.write_json(output_file, flattened, False)
# Also write the .isolate file.
# First, get all the files from all results. Use a map to remove dupes.
files = {}
for item in results_processed.itervalues():
files.update((f.full_path, f) for f in item['results'].existent)
# Convert back to a list, discard the keys.
files = files.values()
value = isolate_common.generate_isolate(files, root_dir, variables,
cwd_dir)
with open('%s.isolate' % output_file, 'wb') as f:
isolate_common.pretty_print(value, f)
return 0
def trace_test_cases(executable, root_dir, cwd_dir, variables, test_cases, jobs, output_file):
"""Traces test cases one by one."""
if not test_cases:
return 0
# Resolve any symlink.
root_dir = os.path.realpath(root_dir)
full_cwd_dir = os.path.join(root_dir, cwd_dir)
logname = output_file + ".logs"
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
api = trace_inputs.get_api()
api.clean_trace(logname)
with api.get_tracer(logname) as tracer:
function = Tracer(tracer, executable, full_cwd_dir, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
values = pool.join(progress, 0.1)
print ""
print "%.1fs Done post-processing logs. Parsing logs." % (time.time() - progress.start)
results = api.parse_log(logname, isolate_common.default_blacklist)
print "%.1fs Done parsing logs." % (time.time() - progress.start)
# Strips to root_dir.
results_processed = {}
for item in results:
if "results" in item:
item = item.copy()
item["results"] = item["results"].strip_root(root_dir)
results_processed[item["trace"]] = item
else:
print >> sys.stderr, "Got exception while tracing %s: %s" % (item["trace"], item["exception"])
print "%.1fs Done stripping root." % (time.time() - progress.start)
# Flatten.
flattened = {}
for item_list in values:
for item in item_list:
if item["valid"]:
test_case = item["test_case"]
tracename = test_case.replace("/", "-")
flattened[test_case] = results_processed[tracename].copy()
item_results = flattened[test_case]["results"]
flattened[test_case].update(
{
"processes": len(list(item_results.process.all)),
"results": item_results.flatten(),
"duration": item["duration"],
"returncode": item["returncode"],
"valid": item["valid"],
"variables": isolate_common.generate_simplified(
item_results.existent, root_dir, variables, cwd_dir
),
}
)
del flattened[test_case]["trace"]
print "%.1fs Done flattening." % (time.time() - progress.start)
# Make it dense if there is more than 20 results.
trace_inputs.write_json(output_file, flattened, False)
# Also write the .isolate file.
# First, get all the files from all results. Use a map to remove dupes.
files = {}
for item in results_processed.itervalues():
files.update((f.full_path, f) for f in item["results"].existent)
# Convert back to a list, discard the keys.
files = files.values()
value = isolate_common.generate_isolate(files, root_dir, variables, cwd_dir)
with open("%s.isolate" % output_file, "wb") as f:
isolate_common.pretty_print(value, f)
return 0
