Exemple #1
0
def process_test_results(results, num_tests, options):
    pb = NullProgressBar()
    if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
        fmt = [
            {'value': 'PASS',    'color': 'green'},
            {'value': 'FAIL',    'color': 'red'},
            {'value': 'TIMEOUT', 'color': 'blue'},
            {'value': 'SKIP',    'color': 'brightgray'},
        ]
        pb = ProgressBar(num_tests, fmt)

    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'
    try:
        for i, res in enumerate(results):

            if options.show_output:
                sys.stdout.write(res.out)
                sys.stdout.write(res.err)
                sys.stdout.write('Exit code: %s\n' % res.rc)
            if res.test.valgrind:
                sys.stdout.write(res.err)

            ok = check_output(res.out, res.err, res.rc, res.test)
            doing = 'after %s' % res.test.path
            if not ok:
                failures.append(res)
                if res.timed_out:
                    pb.message("TIMEOUT - %s" % res.test.path)
                    timeouts += 1
                else:
                    pb.message("FAIL - %s" % res.test.path)

            if options.tinderbox:
                if ok:
                    print_tinderbox("TEST-PASS", res.test);
                else:
                    lines = [ _ for _ in res.out.split('\n') + res.err.split('\n')
                              if _ != '' ]
                    if len(lines) >= 1:
                        msg = lines[-1]
                    else:
                        msg = ''
                    print_tinderbox("TEST-UNEXPECTED-FAIL", res.test, msg);

            n = i + 1
            pb.update(n, {
                'PASS': n - len(failures),
                'FAIL': len(failures),
                'TIMEOUT': timeouts,
                'SKIP': 0}
            )
        complete = True
    except KeyboardInterrupt:
        print_tinderbox("TEST-UNEXPECTED-FAIL", None, "Test execution interrupted by user");

    pb.finish(True)
    return print_test_summary(failures, complete, doing, options)
Exemple #2
0
def process_test_results(results, num_tests, options):
    pb = NullProgressBar()
    if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
        fmt = [
            {'value': 'PASS',    'color': 'green'},
            {'value': 'FAIL',    'color': 'red'},
            {'value': 'TIMEOUT', 'color': 'blue'},
            {'value': 'SKIP',    'color': 'brightgray'},
        ]
        pb = ProgressBar(num_tests, fmt)

    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'
    try:
        for i, res in enumerate(results):

            if options.show_output:
                sys.stdout.write(res.out)
                sys.stdout.write(res.err)
                sys.stdout.write('Exit code: %s\n' % res.rc)
            if res.test.valgrind:
                sys.stdout.write(res.err)

            ok = check_output(res.out, res.err, res.rc, res.test)
            doing = 'after %s' % res.test.path
            if not ok:
                failures.append(res)
                if res.timed_out:
                    pb.message("TIMEOUT - %s" % res.test.path)
                    timeouts += 1
                else:
                    pb.message("FAIL - %s" % res.test.path)

            if options.tinderbox:
                if ok:
                    print_tinderbox("TEST-PASS", res.test);
                else:
                    lines = [ _ for _ in res.out.split('\n') + res.err.split('\n')
                              if _ != '' ]
                    if len(lines) >= 1:
                        msg = lines[-1]
                    else:
                        msg = ''
                    print_tinderbox("TEST-UNEXPECTED-FAIL", res.test, msg);

            n = i + 1
            pb.update(n, {
                'PASS': n - len(failures),
                'FAIL': len(failures),
                'TIMEOUT': timeouts,
                'SKIP': 0}
            )
        complete = True
    except KeyboardInterrupt:
        print_tinderbox("TEST-UNEXPECTED-FAIL", None, "Test execution interrupted by user");

    pb.finish(True)
    return print_test_summary(num_tests, failures, complete, doing, options)
Exemple #3
0
def process_test_results(results, num_tests, options):
    pb = NullProgressBar()
    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'

    if num_tests == 0:
        pb.finish(True)
        complete = True
        return print_test_summary(num_tests, failures, complete, doing, options)

    if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
        fmt = [
            {'value': 'PASS',    'color': 'green'},
            {'value': 'FAIL',    'color': 'red'},
            {'value': 'TIMEOUT', 'color': 'blue'},
            {'value': 'SKIP',    'color': 'brightgray'},
        ]
        pb = ProgressBar(num_tests, fmt)

    try:
        for i, res in enumerate(results):
            if options.show_output:
                sys.stdout.write(res.out)
                sys.stdout.write(res.err)
                sys.stdout.write('Exit code: %s\n' % res.rc)
            if res.test.valgrind:
                sys.stdout.write(res.err)

            ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test)
            doing = 'after %s' % res.test.relpath_tests
            if not ok:
                failures.append(res)
                if res.timed_out:
                    pb.message("TIMEOUT - %s" % res.test.relpath_tests)
                    timeouts += 1
                else:
                    pb.message("FAIL - %s" % res.test.relpath_tests)

            if options.tinderbox:
                print_tinderbox(ok, res)

            n = i + 1
            pb.update(n, {
                'PASS': n - len(failures),
                'FAIL': len(failures),
                'TIMEOUT': timeouts,
                'SKIP': 0}
            )
        complete = True
    except KeyboardInterrupt:
        print("TEST-UNEXPECTED-FAIL | jit_test.py" +
              " : Test execution interrupted by user")

    pb.finish(True)
    return print_test_summary(num_tests, failures, complete, doing, options)
Exemple #4
0
def process_test_results(results, num_tests, options):
    pb = NullProgressBar()
    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'

    if num_tests == 0:
        pb.finish(True)
        complete = True
        return print_test_summary(num_tests, failures, complete, doing, options)

    if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
        fmt = [
            {'value': 'PASS',    'color': 'green'},
            {'value': 'FAIL',    'color': 'red'},
            {'value': 'TIMEOUT', 'color': 'blue'},
            {'value': 'SKIP',    'color': 'brightgray'},
        ]
        pb = ProgressBar(num_tests, fmt)

    try:
        for i, res in enumerate(results):
            if options.show_output:
                sys.stdout.write(res.out)
                sys.stdout.write(res.err)
                sys.stdout.write('Exit code: %s\n' % res.rc)
            if res.test.valgrind:
                sys.stdout.write(res.err)

            ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test)
            doing = 'after %s' % res.test.relpath_tests
            if not ok:
                failures.append(res)
                if res.timed_out:
                    pb.message("TIMEOUT - %s" % res.test.relpath_tests)
                    timeouts += 1
                else:
                    pb.message("FAIL - %s" % res.test.relpath_tests)

            if options.tinderbox:
                print_tinderbox(ok, res)

            n = i + 1
            pb.update(n, {
                'PASS': n - len(failures),
                'FAIL': len(failures),
                'TIMEOUT': timeouts,
                'SKIP': 0}
            )
        complete = True
    except KeyboardInterrupt:
        print("TEST-UNEXPECTED-FAIL | jit_test.py" +
              " : Test execution interrupted by user")

    pb.finish(True)
    return print_test_summary(num_tests, failures, complete, doing, options)
def process_test_results(results, num_tests, options):
    pb = NullProgressBar()
    failures = []
    timeouts = 0
    complete = False
    doing = "before starting"

    if num_tests == 0:
        pb.finish(True)
        complete = True
        return print_test_summary(num_tests, failures, complete, doing, options)

    if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
        fmt = [
            {"value": "PASS", "color": "green"},
            {"value": "FAIL", "color": "red"},
            {"value": "TIMEOUT", "color": "blue"},
            {"value": "SKIP", "color": "brightgray"},
        ]
        pb = ProgressBar(num_tests, fmt)

    try:
        for i, res in enumerate(results):
            if options.show_output:
                sys.stdout.write(res.out)
                sys.stdout.write(res.err)
                sys.stdout.write("Exit code: %s\n" % res.rc)
            if res.test.valgrind:
                sys.stdout.write(res.err)

            ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test)
            doing = "after %s" % res.test.relpath_tests
            if not ok:
                failures.append(res)
                if res.timed_out:
                    pb.message("TIMEOUT - %s" % res.test.relpath_tests)
                    timeouts += 1
                else:
                    pb.message("FAIL - %s" % res.test.relpath_tests)

            if options.tinderbox:
                print_tinderbox(ok, res)

            n = i + 1
            pb.update(n, {"PASS": n - len(failures), "FAIL": len(failures), "TIMEOUT": timeouts, "SKIP": 0})
        complete = True
    except KeyboardInterrupt:
        print("TEST-UNEXPECTED-FAIL | jit_test.py" + " : Test execution interrupted by user")

    pb.finish(True)
    return print_test_summary(num_tests, failures, complete, doing, options)
Exemple #6
0
def process_test_results(results, num_tests, options, lib_dir, shell_args):
    pb = NullProgressBar()
    if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty():
        fmt = [
            {'value': 'PASS',    'color': 'green'},
            {'value': 'FAIL',    'color': 'red'},
            {'value': 'TIMEOUT', 'color': 'blue'},
            {'value': 'SKIP',    'color': 'brightgray'},
        ]
        pb = ProgressBar(num_tests, fmt)

    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'
    try:
        for i, (ok, out, err, code, timed_out, test) in enumerate(results):
            doing = 'after %s'%test.path

            if not ok:
                failures.append([ test, out, err, code, timed_out ])
                pb.message("FAIL - %s" % test.path)
            if timed_out:
                timeouts += 1

            if options.tinderbox:
                if ok:
                    print_tinderbox("TEST-PASS", test);
                else:
                    lines = [ _ for _ in out.split('\n') + err.split('\n')
                              if _ != '' ]
                    if len(lines) >= 1:
                        msg = lines[-1]
                    else:
                        msg = ''
                    print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg);

            n = i + 1
            pb.update(n, {
                'PASS': n - len(failures),
                'FAIL': len(failures),
                'TIMEOUT': timeouts,
                'SKIP': 0}
            )
        complete = True
    except KeyboardInterrupt:
        print_tinderbox("TEST-UNEXPECTED-FAIL", None, "Test execution interrupted by user");

    pb.finish(True)
    return print_test_summary(failures, complete, doing, options, lib_dir, shell_args)
Exemple #7
0
def process_test_results(results, num_tests, options, lib_dir, shell_args):
    pb = NullProgressBar()
    if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty(
    ):
        fmt = [
            {
                'value': 'PASS',
                'color': 'green'
            },
            {
                'value': 'FAIL',
                'color': 'red'
            },
            {
                'value': 'TIMEOUT',
                'color': 'blue'
            },
            {
                'value': 'SKIP',
                'color': 'brightgray'
            },
        ]
        pb = ProgressBar(num_tests, fmt)

    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'
    try:
        for i, (ok, out, err, code, timed_out, test) in enumerate(results):
            doing = 'after %s' % test.path

            if not ok:
                failures.append([test, out, err, code, timed_out])
                pb.message("FAIL - %s" % test.path)
            if timed_out:
                timeouts += 1

            if options.tinderbox:
                if ok:
                    print_tinderbox("TEST-PASS", test)
                else:
                    lines = [
                        _ for _ in out.split('\n') + err.split('\n') if _ != ''
                    ]
                    if len(lines) >= 1:
                        msg = lines[-1]
                    else:
                        msg = ''
                    print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg)

            n = i + 1
            pb.update(
                n, {
                    'PASS': n - len(failures),
                    'FAIL': len(failures),
                    'TIMEOUT': timeouts,
                    'SKIP': 0
                })
        complete = True
    except KeyboardInterrupt:
        print_tinderbox("TEST-UNEXPECTED-FAIL", None,
                        "Test execution interrupted by user")

    pb.finish(True)
    return print_test_summary(failures, complete, doing, options, lib_dir,
                              shell_args)
Exemple #8
0
def run_tests(tests, test_dir, lib_dir, shell_args):
    pb = NullProgressBar()
    if not OPTIONS.hide_progress and not OPTIONS.show_cmd and ProgressBar.conservative_isatty(
    ):
        fmt = [
            {
                'value': 'PASS',
                'color': 'green'
            },
            {
                'value': 'FAIL',
                'color': 'red'
            },
            {
                'value': 'TIMEOUT',
                'color': 'blue'
            },
            {
                'value': 'SKIP',
                'color': 'brightgray'
            },
        ]
        pb = ProgressBar(len(tests), fmt)

    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'
    try:
        for i, test in enumerate(tests):
            doing = 'on %s' % test.path
            ok, out, err, code, timed_out = run_test(test, lib_dir, shell_args)
            doing = 'after %s' % test.path

            if not ok:
                failures.append([test, out, err, code, timed_out])
                pb.message("FAIL - %s" % test.path)
            if timed_out:
                timeouts += 1

            if OPTIONS.tinderbox:
                if ok:
                    print_tinderbox("TEST-PASS", test)
                else:
                    lines = [
                        _ for _ in out.split('\n') + err.split('\n') if _ != ''
                    ]
                    if len(lines) >= 1:
                        msg = lines[-1]
                    else:
                        msg = ''
                    print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg)

            n = i + 1
            pb.update(
                n, {
                    'PASS': n - len(failures),
                    'FAIL': len(failures),
                    'TIMEOUT': timeouts,
                    'SKIP': 0
                })
        complete = True
    except KeyboardInterrupt:
        print_tinderbox("TEST-UNEXPECTED-FAIL", test)

    pb.finish(True)

    if failures:
        if OPTIONS.write_failures:
            try:
                out = open(OPTIONS.write_failures, 'w')
                # Don't write duplicate entries when we are doing multiple failures per job.
                written = set()
                for test, fout, ferr, fcode, _ in failures:
                    if test.path not in written:
                        out.write(os.path.relpath(test.path, test_dir) + '\n')
                        if OPTIONS.write_failure_output:
                            out.write(fout)
                            out.write(ferr)
                            out.write('Exit code: ' + str(fcode) + "\n")
                        written.add(test.path)
                out.close()
            except IOError:
                sys.stderr.write(
                    "Exception thrown trying to write failure file '%s'\n" %
                    OPTIONS.write_failures)
                traceback.print_exc()
                sys.stderr.write('---\n')

        def show_test(test):
            if OPTIONS.show_failed:
                print(
                    '    ' + subprocess.list2cmdline(
                        get_test_cmd(test.path, test.jitflags, lib_dir,
                                     shell_args)))
            else:
                print('    ' + ' '.join(test.jitflags + [test.path]))

        print('FAILURES:')
        for test, _, __, ___, timed_out in failures:
            if not timed_out:
                show_test(test)

        print('TIMEOUTS:')
        for test, _, __, ___, timed_out in failures:
            if timed_out:
                show_test(test)

        return False
    else:
        print('PASSED ALL' +
              ('' if complete else ' (partial run -- interrupted by user %s)' %
               doing))
        return True
Exemple #9
0
def run_tests(tests, test_dir, lib_dir, shell_args):
    pb = NullProgressBar()
    if not OPTIONS.hide_progress and not OPTIONS.show_cmd and ProgressBar.conservative_isatty():
        fmt = [
            {'value': 'PASS',    'color': 'green'},
            {'value': 'FAIL',    'color': 'red'},
            {'value': 'TIMEOUT', 'color': 'blue'},
            {'value': 'SKIP',    'color': 'brightgray'},
        ]
        pb = ProgressBar(len(tests), fmt)

    failures = []
    timeouts = 0
    complete = False
    doing = 'before starting'
    try:
        for i, test in enumerate(tests):
            doing = 'on %s'%test.path
            ok, out, err, code, timed_out = run_test(test, lib_dir, shell_args)
            doing = 'after %s'%test.path

            if not ok:
                failures.append([ test, out, err, code, timed_out ])
                pb.message("FAIL - %s" % test.path)
            if timed_out:
                timeouts += 1

            if OPTIONS.tinderbox:
                if ok:
                    print_tinderbox("TEST-PASS", test);
                else:
                    lines = [ _ for _ in out.split('\n') + err.split('\n')
                              if _ != '' ]
                    if len(lines) >= 1:
                        msg = lines[-1]
                    else:
                        msg = ''
                    print_tinderbox("TEST-UNEXPECTED-FAIL", test, msg);

            n = i + 1
            pb.update(n, {
                'PASS': n - len(failures),
                'FAIL': len(failures),
                'TIMEOUT': timeouts,
                'SKIP': 0}
            )
        complete = True
    except KeyboardInterrupt:
        print_tinderbox("TEST-UNEXPECTED-FAIL", test);

    pb.finish(True)

    if failures:
        if OPTIONS.write_failures:
            try:
                out = open(OPTIONS.write_failures, 'w')
                # Don't write duplicate entries when we are doing multiple failures per job.
                written = set()
                for test, fout, ferr, fcode, _ in failures:
                    if test.path not in written:
                        out.write(os.path.relpath(test.path, test_dir) + '\n')
                        if OPTIONS.write_failure_output:
                            out.write(fout)
                            out.write(ferr)
                            out.write('Exit code: ' + str(fcode) + "\n")
                        written.add(test.path)
                out.close()
            except IOError:
                sys.stderr.write("Exception thrown trying to write failure file '%s'\n"%
                                 OPTIONS.write_failures)
                traceback.print_exc()
                sys.stderr.write('---\n')

        def show_test(test):
            if OPTIONS.show_failed:
                print('    ' + subprocess.list2cmdline(get_test_cmd(test.path, test.jitflags, lib_dir, shell_args)))
            else:
                print('    ' + ' '.join(test.jitflags + [ test.path ]))

        print('FAILURES:')
        for test, _, __, ___, timed_out in failures:
            if not timed_out:
                show_test(test)

        print('TIMEOUTS:')
        for test, _, __, ___, timed_out in failures:
            if timed_out:
                show_test(test)

        return False
    else:
        print('PASSED ALL' + ('' if complete else ' (partial run -- interrupted by user %s)'%doing))
        return True