def test_normal(self):
mapper = lambda value: -value
with worker_pool.ThreadPool(8) as pool:
for i in range(32):
pool.add_task(mapper, i)
results = pool.join()
self.assertEquals(range(-31, 1), sorted(results))
def run_test_cases(executable, whitelist, blacklist, jobs, timeout, stats_only,
no_dump):
"""Traces test cases one by one."""
test_cases = get_test_cases(executable, whitelist, blacklist)
if not test_cases:
return
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
function = Runner(executable, os.getcwd(), timeout, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
results = pool.join(progress, 0.1)
duration = time.time() - progress.start
results = dict((item[0]['test_case'], item) for item in results)
if not no_dump:
trace_inputs.write_json('%s.run_test_cases' % executable, results,
False)
sys.stderr.write('\n')
total = len(results)
if not total:
return 1
# Classify the results
success = []
flaky = []
fail = []
nb_runs = 0
for test_case in sorted(results):
items = results[test_case]
nb_runs += len(items)
if not any(not i['returncode'] for i in items):
fail.append(test_case)
elif len(items) > 1 and any(not i['returncode'] for i in items):
flaky.append(test_case)
elif len(items) == 1 and items[0]['returncode'] == 0:
success.append(test_case)
else:
assert False, items
if not stats_only:
for test_case in sorted(fail):
# Failed, print the last one:
items = results[test_case]
print items[-1]['output']
for test_case in sorted(flaky):
items = results[test_case]
print '%s is flaky (tried %d times)' % (test_case, len(items))
print 'Success: %4d %5.2f%%' % (len(success), len(success) * 100. / total)
print 'Flaky: %4d %5.2f%%' % (len(flaky), len(flaky) * 100. / total)
print 'Fail: %4d %5.2f%%' % (len(fail), len(fail) * 100. / total)
print '%.1fs Done running %d tests with %d executions. %.1f test/s' % (
duration, len(results), nb_runs, nb_runs / duration)
return 0
def test_exception(self):
class FearsomeException(Exception):
pass
def mapper(value):
raise FearsomeException(value)
task_added = False
try:
with worker_pool.ThreadPool(8) as pool:
pool.add_task(mapper, 0)
task_added = True
pool.join()
self.fail()
except FearsomeException:
self.assertEquals(True, task_added)
def trace_test_cases(executable, root_dir, cwd_dir, variables, whitelist,
blacklist, jobs, index, shards, output_file):
"""Traces test cases one by one."""
test_cases = get_test_cases(executable, whitelist, blacklist, index,
shards)
if not test_cases:
return
# Resolve any symlink.
root_dir = os.path.realpath(root_dir)
full_cwd_dir = os.path.join(root_dir, cwd_dir)
logname = output_file + '.logs'
progress = worker_pool.Progress(len(test_cases))
with worker_pool.ThreadPool(jobs or multiprocessing.cpu_count()) as pool:
api = trace_inputs.get_api()
api.clean_trace(logname)
with api.get_tracer(logname) as tracer:
function = Tracer(tracer, executable, full_cwd_dir, progress).map
for test_case in test_cases:
pool.add_task(function, test_case)
values = pool.join(progress, 0.1)
print ''
print '%.1fs Done post-processing logs. Parsing logs.' % (time.time() -
progress.start)
results = api.parse_log(logname, isolate_common.default_blacklist)
print '%.1fs Done parsing logs.' % (time.time() - progress.start)
# Strips to root_dir.
results_processed = {}
for item in results:
if 'results' in item:
item = item.copy()
item['results'] = item['results'].strip_root(root_dir)
results_processed[item['trace']] = item
else:
print >> sys.stderr, 'Got exception while tracing %s: %s' % (
item['trace'], item['exception'])
print '%.1fs Done stripping root.' % (time.time() - progress.start)
# Flatten.
flattened = {}
for item_list in values:
for item in item_list:
if item['valid']:
test_case = item['test_case']
tracename = test_case.replace('/', '-')
flattened[test_case] = results_processed[tracename].copy()
item_results = flattened[test_case]['results']
flattened[test_case].update({
'processes':
len(list(item_results.process.all)),
'results':
item_results.flatten(),
'duration':
item['duration'],
'returncode':
item['returncode'],
'valid':
item['valid'],
'variables':
isolate_common.generate_simplified(item_results.existent,
root_dir, variables,
cwd_dir),
})
del flattened[test_case]['trace']
print '%.1fs Done flattening.' % (time.time() - progress.start)
# Make it dense if there is more than 20 results.
trace_inputs.write_json(output_file, flattened, False)
# Also write the .isolate file.
# First, get all the files from all results. Use a map to remove dupes.
files = {}
for item in results_processed.itervalues():
files.update((f.full_path, f) for f in item['results'].existent)
# Convert back to a list, discard the keys.
files = files.values()
value = isolate_common.generate_isolate(files, root_dir, variables,
cwd_dir)
with open('%s.isolate' % output_file, 'wb') as f:
isolate_common.pretty_print(value, f)
return 0
def test_trace_multiple(self):
# Starts PARALLEL threads and trace PARALLEL child processes simultaneously.
# Some are started from 'data' directory, others from this script's
# directory. One trace fails. Verify everything still goes one.
PARALLEL = 8
def trace(tracer, cmd, cwd, tracename):
resultcode, output = tracer.trace(
cmd, cwd, tracename, True)
return (tracename, resultcode, output)
with worker_pool.ThreadPool(PARALLEL) as pool:
api = self.trace_inputs.get_api()
with api.get_tracer(self.log) as tracer:
pool.add_task(
trace, tracer, self.get_child_command(False), ROOT_DIR, 'trace1')
pool.add_task(
trace, tracer, self.get_child_command(True), self.cwd, 'trace2')
pool.add_task(
trace, tracer, self.get_child_command(False), ROOT_DIR, 'trace3')
pool.add_task(
trace, tracer, self.get_child_command(True), self.cwd, 'trace4')
# Have this one fail since it's started from the wrong directory.
pool.add_task(
trace, tracer, self.get_child_command(False), self.cwd, 'trace5')
pool.add_task(
trace, tracer, self.get_child_command(True), self.cwd, 'trace6')
pool.add_task(
trace, tracer, self.get_child_command(False), ROOT_DIR, 'trace7')
pool.add_task(
trace, tracer, self.get_child_command(True), self.cwd, 'trace8')
trace_results = pool.join()
def blacklist(f):
return f.endswith('.pyc')
actual_results = api.parse_log(self.log, blacklist)
self.assertEquals(8, len(trace_results))
self.assertEquals(8, len(actual_results))
# Convert to dict keyed on the trace name, simpler to verify.
trace_results = dict((i[0], i[1:]) for i in trace_results)
actual_results = dict((x.pop('trace'), x) for x in actual_results)
self.assertEquals(sorted(trace_results), sorted(actual_results))
# It'd be nice to start different kinds of processes.
expected_results = [
self._gen_dict_full(),
self._gen_dict_full_gyp(),
self._gen_dict_full(),
self._gen_dict_full_gyp(),
self._gen_dict_wrong_path(),
self._gen_dict_full_gyp(),
self._gen_dict_full(),
self._gen_dict_full_gyp(),
]
self.assertEquals(len(expected_results), len(trace_results))
# See the comment above about the trace that fails because it's started from
# the wrong directory.
BUSTED = 4
for index, key in enumerate(sorted(actual_results)):
self.assertEquals('trace%d' % (index + 1), key)
self.assertEquals(2, len(trace_results[key]))
# returncode
self.assertEquals(0 if index != BUSTED else 2, trace_results[key][0])
# output
self.assertEquals(actual_results[key]['output'], trace_results[key][1])
self.assertEquals(['output', 'results'], sorted(actual_results[key]))
results = actual_results[key]['results']
results = results.strip_root(ROOT_DIR)
actual = results.flatten()
self.assertTrue(actual['root'].pop('pid'))
if index != BUSTED:
self.assertTrue(actual['root']['children'][0].pop('pid'))
self.assertEquals(expected_results[index], actual)