def process_test_results(results, num_tests, options):
pb = NullProgressBar()
if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
pb = ProgressBar(num_tests, fmt)
failures = []
timeouts = 0
complete = False
doing = 'before starting'
try:
for i, res in enumerate(results):
if options.show_output:
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write('Exit code: %s\n' % res.rc)
if res.test.valgrind:
sys.stdout.write(res.err)
ok = check_output(res.out, res.err, res.rc, res.test)
doing = 'after %s' % res.test.path
if not ok:
failures.append(res)
if res.timed_out:
pb.message("TIMEOUT - %s" % res.test.path)
timeouts += 1
else:
pb.message("FAIL - %s" % res.test.path)
if options.tinderbox:
if ok:
print_tinderbox("TEST-PASS", res.test);
else:
lines = [ _ for _ in res.out.split('\n') + res.err.split('\n')
if _ != '' ]
if len(lines) >= 1:
msg = lines[-1]
else:
msg = ''
print_tinderbox("TEST-UNEXPECTED-FAIL", res.test, msg);
n = i + 1
pb.update(n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0}
)
complete = True
except KeyboardInterrupt:
print_tinderbox("TEST-UNEXPECTED-FAIL", None, "Test execution interrupted by user");
pb.finish(True)
return print_test_summary(failures, complete, doing, options)
def process_test_results(results, num_tests, options):
pb = NullProgressBar()
if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
pb = ProgressBar(num_tests, fmt)
failures = []
timeouts = 0
complete = False
doing = 'before starting'
try:
for i, res in enumerate(results):
if options.show_output:
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write('Exit code: %s\n' % res.rc)
if res.test.valgrind:
sys.stdout.write(res.err)
ok = check_output(res.out, res.err, res.rc, res.test)
doing = 'after %s' % res.test.path
if not ok:
failures.append(res)
pb.message("FAIL - %s" % res.test.path)
if res.timed_out:
timeouts += 1
if options.tinderbox:
if ok:
print_tinderbox("TEST-PASS", res.test);
else:
lines = [ _ for _ in res.out.split('\n') + res.err.split('\n')
if _ != '' ]
if len(lines) >= 1:
msg = lines[-1]
else:
msg = ''
print_tinderbox("TEST-UNEXPECTED-FAIL", res.test, msg);
n = i + 1
pb.update(n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0}
)
complete = True
except KeyboardInterrupt:
print_tinderbox("TEST-UNEXPECTED-FAIL", None, "Test execution interrupted by user");
pb.finish(True)
return print_test_summary(failures, complete, doing, options)
def __init__(self, options, testcount):
self.options = options
self.fp = options.output_fp
self.groups = {}
self.output_dict = {}
self.counts = {'PASS': 0, 'FAIL': 0, 'TIMEOUT': 0, 'SKIP': 0}
self.slow_tests = []
self.n = 0
if options.hide_progress:
self.pb = NullProgressBar()
else:
fmt = [
{
'value': 'PASS',
'color': 'green'
},
{
'value': 'FAIL',
'color': 'red'
},
{
'value': 'TIMEOUT',
'color': 'blue'
},
{
'value': 'SKIP',
'color': 'brightgray'
},
]
self.pb = ProgressBar(testcount, fmt)
def __init__(self, testsuite, options, testcount):
self.options = options
self.fp = options.output_fp
if self.options.format == 'automation':
self.slog = TestLogger(testsuite)
self.slog.suite_start()
self.wptreport = None
if self.options.wptreport:
try:
from .wptreport import WptreportHandler
self.wptreport = WptreportHandler(self.options.wptreport)
self.wptreport.suite_start()
except ImportError:
pass
self.groups = {}
self.output_dict = {}
self.counts = {'PASS': 0, 'FAIL': 0, 'TIMEOUT': 0, 'SKIP': 0}
self.slow_tests = []
self.n = 0
if options.hide_progress:
self.pb = NullProgressBar()
else:
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
self.pb = ProgressBar(testcount, fmt)
def process_test_results(results, num_tests, options):
pb = NullProgressBar()
if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
fmt = [
{"value": "PASS", "color": "green"},
{"value": "FAIL", "color": "red"},
{"value": "TIMEOUT", "color": "blue"},
{"value": "SKIP", "color": "brightgray"},
]
pb = ProgressBar(num_tests, fmt)
failures = []
timeouts = 0
complete = False
doing = "before starting"
try:
for i, res in enumerate(results):
if options.show_output:
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write("Exit code: %s\n" % res.rc)
if res.test.valgrind:
sys.stdout.write(res.err)
ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test)
doing = "after %s" % res.test.relpath_tests
if not ok:
failures.append(res)
if res.timed_out:
pb.message("TIMEOUT - %s" % res.test.relpath_tests)
timeouts += 1
else:
pb.message("FAIL - %s" % res.test.relpath_tests)
if options.tinderbox:
print_tinderbox(ok, res)
n = i + 1
pb.update(n, {"PASS": n - len(failures), "FAIL": len(failures), "TIMEOUT": timeouts, "SKIP": 0})
complete = True
except KeyboardInterrupt:
print("TEST-UNEXPECTED-FAIL | jit_test.py" + " : Test execution interrupted by user")
pb.finish(True)
return print_test_summary(num_tests, failures, complete, doing, options)
def create_progressbar(num_tests, options):
if not options.hide_progress and not options.show_cmd \
and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
return ProgressBar(num_tests, fmt)
return NullProgressBar()
def process_test_results(results, num_tests, options, lib_dir, shell_args):
pb = NullProgressBar()
if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
pb = ProgressBar(num_tests, fmt)
failures = []
timeouts = 0
complete = False
doing = 'before starting'
try:
for i, (ok, out, err, code, timed_out, test) in enumerate(results):
doing = 'after %s'%test.path
if not ok:
failures.append([ test, out, err, code, timed_out ])
pb.message("FAIL - %s" % test.path)
if timed_out:
timeouts += 1
if options.tinderbox:
if ok:
print_tinderbox("TEST-PASS", test);
else:
lines = [ _ for _ in out.split('\n') + err.split('\n')
if _ != '' ]
if len(lines) >= 1:
msg = lines[-1]
else:
msg = ''
print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg);
n = i + 1
pb.update(n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0}
)
complete = True
except KeyboardInterrupt:
print_tinderbox("TEST-UNEXPECTED-FAIL", None, "Test execution interrupted by user");
pb.finish(True)
return print_test_summary(failures, complete, doing, options, lib_dir, shell_args)
def process_test_results(results, num_tests, options):
pb = NullProgressBar()
failures = []
timeouts = 0
complete = False
doing = 'before starting'
if num_tests == 0:
pb.finish(True)
complete = True
return print_test_summary(num_tests, failures, complete, doing, options)
if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
pb = ProgressBar(num_tests, fmt)
try:
for i, res in enumerate(results):
if options.show_output:
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write('Exit code: %s\n' % res.rc)
if res.test.valgrind:
sys.stdout.write(res.err)
ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test)
doing = 'after %s' % res.test.relpath_tests
if not ok:
failures.append(res)
if res.timed_out:
pb.message("TIMEOUT - %s" % res.test.relpath_tests)
timeouts += 1
else:
pb.message("FAIL - %s" % res.test.relpath_tests)
if options.tinderbox:
print_tinderbox(ok, res)
n = i + 1
pb.update(n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0}
)
complete = True
except KeyboardInterrupt:
print("TEST-UNEXPECTED-FAIL | jit_test.py" +
" : Test execution interrupted by user")
pb.finish(True)
return print_test_summary(num_tests, failures, complete, doing, options)
def process_test_results(results, num_tests, options):
pb = NullProgressBar()
failures = []
timeouts = 0
complete = False
doing = 'before starting'
if num_tests == 0:
pb.finish(True)
complete = True
return print_test_summary(num_tests, failures, complete, doing, options)
if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
pb = ProgressBar(num_tests, fmt)
try:
for i, res in enumerate(results):
ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test, options)
if ok:
show_output = options.show_output and not options.failed_only
else:
show_output = options.show_output or not options.no_show_failed
if show_output:
pb.beginline()
sys.stdout.write(res.out)
sys.stdout.write(res.err)
sys.stdout.write('Exit code: %s\n' % res.rc)
if res.test.valgrind and not show_output:
pb.beginline()
sys.stdout.write(res.err)
doing = 'after %s' % res.test.relpath_tests
if not ok:
failures.append(res)
if res.timed_out:
pb.message("TIMEOUT - %s" % res.test.relpath_tests)
timeouts += 1
else:
pb.message("FAIL - %s" % res.test.relpath_tests)
if options.tinderbox:
print_tinderbox(ok, res)
n = i + 1
pb.update(n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0}
)
complete = True
except KeyboardInterrupt:
print("TEST-UNEXPECTED-FAIL | jit_test.py" +
" : Test execution interrupted by user")
pb.finish(True)
return print_test_summary(num_tests, failures, complete, doing, options)
def process_test_results(results, num_tests, options, lib_dir, shell_args):
pb = NullProgressBar()
if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty(
):
fmt = [
{
'value': 'PASS',
'color': 'green'
},
{
'value': 'FAIL',
'color': 'red'
},
{
'value': 'TIMEOUT',
'color': 'blue'
},
{
'value': 'SKIP',
'color': 'brightgray'
},
]
pb = ProgressBar(num_tests, fmt)
failures = []
timeouts = 0
complete = False
doing = 'before starting'
try:
for i, (ok, out, err, code, timed_out, test) in enumerate(results):
doing = 'after %s' % test.path
if not ok:
failures.append([test, out, err, code, timed_out])
pb.message("FAIL - %s" % test.path)
if timed_out:
timeouts += 1
if options.tinderbox:
if ok:
print_tinderbox("TEST-PASS", test)
else:
lines = [
_ for _ in out.split('\n') + err.split('\n') if _ != ''
]
if len(lines) >= 1:
msg = lines[-1]
else:
msg = ''
print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg)
n = i + 1
pb.update(
n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0
})
complete = True
except KeyboardInterrupt:
print_tinderbox("TEST-UNEXPECTED-FAIL", None,
"Test execution interrupted by user")
pb.finish(True)
return print_test_summary(failures, complete, doing, options, lib_dir,
shell_args)
def run_tests(tests, test_dir, lib_dir, shell_args):
pb = NullProgressBar()
if not OPTIONS.hide_progress and not OPTIONS.show_cmd and ProgressBar.conservative_isatty(
):
fmt = [
{
'value': 'PASS',
'color': 'green'
},
{
'value': 'FAIL',
'color': 'red'
},
{
'value': 'TIMEOUT',
'color': 'blue'
},
{
'value': 'SKIP',
'color': 'brightgray'
},
]
pb = ProgressBar(len(tests), fmt)
failures = []
timeouts = 0
complete = False
doing = 'before starting'
try:
for i, test in enumerate(tests):
doing = 'on %s' % test.path
ok, out, err, code, timed_out = run_test(test, lib_dir, shell_args)
doing = 'after %s' % test.path
if not ok:
failures.append([test, out, err, code, timed_out])
pb.message("FAIL - %s" % test.path)
if timed_out:
timeouts += 1
if OPTIONS.tinderbox:
if ok:
print_tinderbox("TEST-PASS", test)
else:
lines = [
_ for _ in out.split('\n') + err.split('\n') if _ != ''
]
if len(lines) >= 1:
msg = lines[-1]
else:
msg = ''
print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg)
n = i + 1
pb.update(
n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0
})
complete = True
except KeyboardInterrupt:
print_tinderbox("TEST-UNEXPECTED-FAIL", test)
pb.finish(True)
if failures:
if OPTIONS.write_failures:
try:
out = open(OPTIONS.write_failures, 'w')
# Don't write duplicate entries when we are doing multiple failures per job.
written = set()
for test, fout, ferr, fcode, _ in failures:
if test.path not in written:
out.write(os.path.relpath(test.path, test_dir) + '\n')
if OPTIONS.write_failure_output:
out.write(fout)
out.write(ferr)
out.write('Exit code: ' + str(fcode) + "\n")
written.add(test.path)
out.close()
except IOError:
sys.stderr.write(
"Exception thrown trying to write failure file '%s'\n" %
OPTIONS.write_failures)
traceback.print_exc()
sys.stderr.write('---\n')
def show_test(test):
if OPTIONS.show_failed:
print(
' ' + subprocess.list2cmdline(
get_test_cmd(test.path, test.jitflags, lib_dir,
shell_args)))
else:
print(' ' + ' '.join(test.jitflags + [test.path]))
print('FAILURES:')
for test, _, __, ___, timed_out in failures:
if not timed_out:
show_test(test)
print('TIMEOUTS:')
for test, _, __, ___, timed_out in failures:
if timed_out:
show_test(test)
return False
else:
print('PASSED ALL' +
('' if complete else ' (partial run -- interrupted by user %s)' %
doing))
return True
def run_tests(tests, test_dir, lib_dir, shell_args):
pb = NullProgressBar()
if not OPTIONS.hide_progress and not OPTIONS.show_cmd and ProgressBar.conservative_isatty():
fmt = [
{'value': 'PASS', 'color': 'green'},
{'value': 'FAIL', 'color': 'red'},
{'value': 'TIMEOUT', 'color': 'blue'},
{'value': 'SKIP', 'color': 'brightgray'},
]
pb = ProgressBar(len(tests), fmt)
failures = []
timeouts = 0
complete = False
doing = 'before starting'
try:
for i, test in enumerate(tests):
doing = 'on %s'%test.path
ok, out, err, code, timed_out = run_test(test, lib_dir, shell_args)
doing = 'after %s'%test.path
if not ok:
failures.append([ test, out, err, code, timed_out ])
pb.message("FAIL - %s" % test.path)
if timed_out:
timeouts += 1
if OPTIONS.tinderbox:
if ok:
print_tinderbox("TEST-PASS", test);
else:
lines = [ _ for _ in out.split('\n') + err.split('\n')
if _ != '' ]
if len(lines) >= 1:
msg = lines[-1]
else:
msg = ''
print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg);
n = i + 1
pb.update(n, {
'PASS': n - len(failures),
'FAIL': len(failures),
'TIMEOUT': timeouts,
'SKIP': 0}
)
complete = True
except KeyboardInterrupt:
print_tinderbox("TEST-UNEXPECTED-FAIL", test);
pb.finish(True)
if failures:
if OPTIONS.write_failures:
try:
out = open(OPTIONS.write_failures, 'w')
# Don't write duplicate entries when we are doing multiple failures per job.
written = set()
for test, fout, ferr, fcode, _ in failures:
if test.path not in written:
out.write(os.path.relpath(test.path, test_dir) + '\n')
if OPTIONS.write_failure_output:
out.write(fout)
out.write(ferr)
out.write('Exit code: ' + str(fcode) + "\n")
written.add(test.path)
out.close()
except IOError:
sys.stderr.write("Exception thrown trying to write failure file '%s'\n"%
OPTIONS.write_failures)
traceback.print_exc()
sys.stderr.write('---\n')
def show_test(test):
if OPTIONS.show_failed:
print(' ' + subprocess.list2cmdline(get_test_cmd(test.path, test.jitflags, lib_dir, shell_args)))
else:
print(' ' + ' '.join(test.jitflags + [ test.path ]))
print('FAILURES:')
for test, _, __, ___, timed_out in failures:
if not timed_out:
show_test(test)
print('TIMEOUTS:')
for test, _, __, ___, timed_out in failures:
if timed_out:
show_test(test)
return False
else:
print('PASSED ALL' + ('' if complete else ' (partial run -- interrupted by user %s)'%doing))
return True
