Example #1
0
def run(args):
    try:
        _opts = load(args = args, defaults = defaults_mc)
        log.notice('RTEMS Test - Defaults, v%s' % (version.str()))
        _opts.log_info()
        log.notice('Options:')
        log.notice(str(_opts))
        log.notice('Defaults:')
        log.notice(str(_opts.defaults))
    except error.general as gerr:
        print(gerr, file = sys.stderr)
        sys.exit(1)
    except error.internal as ierr:
        print(ierr, file = sys.stderr)
        sys.exit(1)
    except error.exit:
        pass
    except KeyboardInterrupt:
        log.notice('abort: user terminated')
        sys.exit(1)
    sys.exit(0)
Example #2
0
def run(args):
    try:
        _opts = load(args=args, defaults=defaults_mc)
        log.notice('RTEMS Test - Defaults, v%s' % (version.str()))
        _opts.log_info()
        log.notice('Options:')
        log.notice(str(_opts))
        log.notice('Defaults:')
        log.notice(str(_opts.defaults))
    except error.general as gerr:
        print(gerr, file=sys.stderr)
        sys.exit(1)
    except error.internal as ierr:
        print(ierr, file=sys.stderr)
        sys.exit(1)
    except error.exit:
        pass
    except KeyboardInterrupt:
        log.notice('abort: user terminated')
        sys.exit(1)
    sys.exit(0)
Example #3
0
def run_args(args):
    b = None
    ec = 0
    try:
        #
        # On Windows MSYS2 prepends a path to itself to the environment
        # path. This means the RTEMS specific automake is not found and which
        # breaks the bootstrap. We need to remove the prepended path. Also
        # remove any ACLOCAL paths from the environment.
        #
        if os.name == 'nt':
            cspath = os.environ['PATH'].split(os.pathsep)
            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
                os.environ['PATH'] = os.pathsep.join(cspath[1:])

        top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
        prefix = '/opt/rtems/%s' % (rtems_version())
        tools = prefix
        build_dir = 'bsp-builds'
        logf = 'bsp-build-%s.txt' % (datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
        config_file = path.join(top, 'share', 'rtems', 'tester', 'rtems', 'rtems-bsps.ini')
        if not path.exists(config_file):
            config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')

        argsp = argparse.ArgumentParser()
        argsp.add_argument('--prefix', help = 'Prefix to build the BSP.', type = str)
        argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.', type = str)
        argsp.add_argument('--rtems', help = 'The RTEMS source tree.', type = str)
        argsp.add_argument('--build-path', help = 'Path to build in.', type = str)
        argsp.add_argument('--log', help = 'Log file.', type = str)
        argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
                           action = 'store_true')
        argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
                           action = 'store_true')
        argsp.add_argument('--profiles', help = 'Build the listed profiles.',
                           type = str, default = 'tier-1')
        argsp.add_argument('--build', help = 'Build variation.', type = str)
        argsp.add_argument('--arch', help = 'Build the specific architecture.', type = str)
        argsp.add_argument('--bsp', help = 'Build the specific BSP.', type = str)
        argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
                           action = 'store_true')

        opts = argsp.parse_args(args[1:])
        if opts.log is not None:
            logf = opts.log
        log.default = log.log([logf])
        log.notice('RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str()))
        if opts.rtems is None:
            raise error.general('No RTEMS source provided on the command line')
        if opts.prefix is not None:
            prefix = path.shell(opts.prefix)
        if opts.rtems_tools is not None:
            tools = path.shell(opts.rtems_tools)
        if opts.build_path is not None:
            build_dir = path.shell(opts.build_path)
        if opts.bsp is not None and opts.arch is None:
            raise error.general('BSP provided but no architecture')

        config = configuration()
        config.load(config_file, opts.build)

        options = { 'stop-on-error' : opts.stop_on_error,
                    'no-clean'      : opts.no_clean,
                    'dry-run'       : opts.dry_run,
                    'jobs'          : 8 }

        b = build(config, rtems_version(), prefix, tools,
                  path.shell(opts.rtems), build_dir, options)
        if opts.arch is not None:
            if opts.bsp is not None:
                b.build_arch_bsp(opts.arch, opts.bsp)
            else:
                b.build_arch(opts.arch)
        else:
            for profile in opts.profiles.split(','):
                b.build_profile(profile.strip())

    except error.general as gerr:
        print(gerr)
        print('BSP Build FAILED', file = sys.stderr)
        ec = 1
    except error.internal as ierr:
        print(ierr)
        print('BSP Build FAILED', file = sys.stderr)
        ec = 1
    except error.exit as eerr:
        pass
    except KeyboardInterrupt:
        log.notice('abort: user terminated')
        ec = 1
    if b is not None:
        b.results.report()
    sys.exit(ec)
Example #4
0
def run(command_path=None):
    import sys
    tests = []
    stdtty = console.save()
    opts = None
    default_exefilter = '*.exe'
    try:
        optargs = {
            '--rtems-tools':
            'The path to the RTEMS tools',
            '--rtems-bsp':
            'The RTEMS BSP to run the test on',
            '--report-mode':
            'Reporting modes, failures (default),all,none',
            '--list-bsps':
            'List the supported BSPs',
            '--debug-trace':
            'Debug trace based on specific flags',
            '--filter':
            'Glob that executables must match to run (default: ' +
            default_exefilter + ')',
            '--stacktrace':
            'Dump a stack trace on a user termination (^C)',
            '--coverage':
            'Perform coverage analysis of test exectuables.',
            '--rtems-builddir':
            'The path to the build directory.'
        }
        opts = options.load(sys.argv,
                            optargs=optargs,
                            command_path=command_path)
        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
        if opts.find_arg('--list-bsps'):
            bsps.list(opts)
        exe_filter = opts.find_arg('--filter')
        if exe_filter:
            exe_filter = exe_filter[1]
        else:
            exe_filter = default_exefilter
        opts.log_info()
        debug_trace = opts.find_arg('--debug-trace')
        if debug_trace:
            if len(debug_trace) != 1:
                debug_trace = debug_trace[1]
            else:
                raise error.general(
                    'no debug flags, can be: console,gdb,output')
        else:
            debug_trace = ''
        opts.defaults['debug_trace'] = debug_trace
        job_trace = 'jobs' in debug_trace.split(',')
        rtems_tools = opts.find_arg('--rtems-tools')
        if rtems_tools:
            if len(rtems_tools) != 2:
                raise error.general('invalid RTEMS tools option')
            rtems_tools = rtems_tools[1]
        else:
            rtems_tools = '%{_prefix}'
        bsp = opts.find_arg('--rtems-bsp')
        if bsp is None or len(bsp) != 2:
            raise error.general('RTEMS BSP not provided or invalid option')
        opts.defaults.load('%%{_configdir}/bsps/%s.mc' % (bsp[1]))
        bsp = opts.defaults.get('%{bsp}')
        if not bsp:
            raise error.general(
                'BSP definition (%{bsp}) not found in the global map')
        bsp = bsp[2]
        if not opts.defaults.set_read_map(bsp):
            raise error.general('no BSP map found')
        bsp_script = opts.defaults.get(bsp)
        if not bsp_script:
            raise error.general('BSP script not found: %s' % (bsp))
        bsp_config = opts.defaults.expand(opts.defaults[bsp])
        path_to_builddir = opts.find_arg('--rtems-builddir')
        if not path_to_builddir:
            raise error.general('Path to build directory not provided')
        coverage_enabled = opts.find_arg('--coverage')
        if coverage_enabled:
            coverage = prepare_coverage(opts, path_to_builddir[1])
            coverage.prepare_environment()
        report_mode = opts.find_arg('--report-mode')
        if report_mode:
            if report_mode[1] != 'failures' and \
                    report_mode[1] != 'all' and \
                    report_mode[1] != 'none':
                raise error.general('invalid report mode')
            report_mode = report_mode[1]
        else:
            report_mode = 'failures'
        executables = find_executables(opts.params(), exe_filter,
                                       path_to_builddir[1])
        if len(executables) == 0:
            raise error.general('no executables supplied')
        start_time = datetime.datetime.now()
        total = len(executables)
        reports = report.report(total)
        reporting = 1
        jobs = int(opts.jobs(opts.defaults['_ncpus']))
        exe = 0
        finished = []
        if jobs > len(executables):
            jobs = len(executables)
        while exe < total or len(tests) > 0:
            if exe < total and len(tests) < jobs:
                tst = test_run(exe + 1, total, reports, executables[exe],
                               rtems_tools, bsp, bsp_config, opts)
                exe += 1
                tests += [tst]
                if job_trace:
                    _job_trace(tst, 'create', total, exe, tests, reporting)
                tst.run()
            else:
                dead = [t for t in tests if not t.is_alive()]
                tests[:] = [t for t in tests if t not in dead]
                for tst in dead:
                    if job_trace:
                        _job_trace(tst, 'dead', total, exe, tests, reporting)
                    finished += [tst]
                    tst.reraise()
                del dead
                if len(tests) >= jobs or exe >= total:
                    time.sleep(0.250)
                if len(finished):
                    reporting = report_finished(reports, report_mode,
                                                reporting, finished, job_trace)
        finished_time = datetime.datetime.now()
        reporting = report_finished(reports, report_mode, reporting, finished,
                                    job_trace)
        if reporting < total:
            log.warning('finished jobs does match: %d' % (reporting))
            report_finished(reports, report_mode, -1, finished, job_trace)
        reports.summary()
        end_time = datetime.datetime.now()
        log.notice('Average test time: %s' % (str(
            (end_time - start_time) / total)))
        log.notice('Testing time     : %s' % (str(end_time - start_time)))
        if coverage_enabled:
            coverage_run(opts, coverage, executables)
    except error.general as gerr:
        print(gerr)
        sys.exit(1)
    except error.internal as ierr:
        print(ierr)
        sys.exit(1)
    except error.exit:
        sys.exit(2)
    except KeyboardInterrupt:
        if opts is not None and opts.find_arg('--stacktrace'):
            print('}} dumping:', threading.active_count())
            for t in threading.enumerate():
                print('}} ', t.name)
            print(stacktraces.trace())
        log.notice('abort: user terminated')
        killall(tests)
        sys.exit(1)
    finally:
        console.restore(stdtty)
    sys.exit(0)
Example #5
0
def run(command_path = None):
    import sys
    tests = []
    stdtty = console.save()
    opts = None
    default_exefilter = '*.exe'
    try:
        optargs = { '--rtems-tools': 'The path to the RTEMS tools',
                    '--rtems-bsp':   'The RTEMS BSP to run the test on',
                    '--report-mode': 'Reporting modes, failures (default),all,none',
                    '--list-bsps':   'List the supported BSPs',
                    '--debug-trace': 'Debug trace based on specific flags',
                    '--filter':      'Glob that executables must match to run (default: ' +
                              default_exefilter + ')',
                    '--stacktrace':  'Dump a stack trace on a user termination (^C)' }
        opts = options.load(sys.argv,
                            optargs = optargs,
                            command_path = command_path)
        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
        if opts.find_arg('--list-bsps'):
            bsps.list(opts)
        exe_filter = opts.find_arg('--filter')
        if exe_filter:
            exe_filter = exe_filter[1]
        else:
            exe_filter = default_exefilter
        opts.log_info()
        debug_trace = opts.find_arg('--debug-trace')
        if debug_trace:
            if len(debug_trace) != 1:
                debug_trace = debug_trace[1]
            else:
                raise error.general('no debug flags, can be: console,gdb,output')
        else:
            debug_trace = ''
        opts.defaults['debug_trace'] = debug_trace
        job_trace = 'jobs' in debug_trace.split(',')
        rtems_tools = opts.find_arg('--rtems-tools')
        if rtems_tools:
            if len(rtems_tools) != 2:
                raise error.general('invalid RTEMS tools option')
            rtems_tools = rtems_tools[1]
        else:
            rtems_tools = '%{_prefix}'
        bsp = opts.find_arg('--rtems-bsp')
        if bsp is None or len(bsp) != 2:
            raise error.general('RTEMS BSP not provided or invalid option')
        opts.defaults.load('%%{_configdir}/bsps/%s.mc' % (bsp[1]))
        bsp = opts.defaults.get('%{bsp}')
        if not bsp:
            raise error.general('BSP definition (%{bsp}) not found in the global map')
        bsp = bsp[2]
        if not opts.defaults.set_read_map(bsp):
            raise error.general('no BSP map found')
        bsp_script = opts.defaults.get(bsp)
        if not bsp_script:
            raise error.general('BSP script not found: %s' % (bsp))
        bsp_config = opts.defaults.expand(opts.defaults[bsp])
        report_mode = opts.find_arg('--report-mode')
        if report_mode:
            if report_mode[1] != 'failures' and \
                    report_mode[1] != 'all' and \
                    report_mode[1] != 'none':
                raise error.general('invalid report mode')
            report_mode = report_mode[1]
        else:
            report_mode = 'failures'
        executables = find_executables(opts.params(), exe_filter)
        if len(executables) == 0:
            raise error.general('no executables supplied')
        start_time = datetime.datetime.now()
        total = len(executables)
        reports = report.report(total)
        invalid_tests = opts.defaults['invalid_tests']
        if invalid_tests:
            reports.set_invalid_tests([l.strip() for l in invalid_tests.splitlines()])
        reporting = 1
        jobs = int(opts.jobs(opts.defaults['_ncpus']))
        exe = 0
        finished = []
        if jobs > len(executables):
            jobs = len(executables)
        while exe < total or len(tests) > 0:
            if exe < total and len(tests) < jobs:
                tst = test_run(exe + 1, total, reports,
                               executables[exe],
                               rtems_tools, bsp, bsp_config,
                               opts)
                exe += 1
                tests += [tst]
                if job_trace:
                    _job_trace(tst, 'create',
                               total, exe, tests, reporting)
                tst.run()
            else:
                dead = [t for t in tests if not t.is_alive()]
                tests[:] = [t for t in tests if t not in dead]
                for tst in dead:
                    if job_trace:
                        _job_trace(tst, 'dead',
                                   total, exe, tests, reporting)
                    finished += [tst]
                    tst.reraise()
                del dead
                if len(tests) >= jobs or exe >= total:
                    time.sleep(0.250)
                if len(finished):
                    reporting = report_finished(reports,
                                                report_mode,
                                                reporting,
                                                finished,
                                                job_trace)
        finished_time = datetime.datetime.now()
        reporting = report_finished(reports, report_mode,
                                    reporting, finished, job_trace)
        if reporting < total:
            log.warning('finished jobs does match: %d' % (reporting))
            report_finished(reports, report_mode, -1, finished, job_trace)
        reports.summary()
        end_time = datetime.datetime.now()
        log.notice('Average test time: %s' % (str((end_time - start_time) / total)))
        log.notice('Testing time     : %s' % (str(end_time - start_time)))
    except error.general as gerr:
        print(gerr)
        sys.exit(1)
    except error.internal as ierr:
        print(ierr)
        sys.exit(1)
    except error.exit:
        sys.exit(2)
    except KeyboardInterrupt:
        if opts is not None and opts.find_arg('--stacktrace'):
            print('}} dumping:', threading.active_count())
            for t in threading.enumerate():
                print('}} ', t.name)
            print(stacktraces.trace())
        log.notice('abort: user terminated')
        killall(tests)
        sys.exit(1)
    finally:
        console.restore(stdtty)
    sys.exit(0)
Example #6
0
def run_args(args):
    b = None
    ec = 0
    try:
        #
        # On Windows MSYS2 prepends a path to itself to the environment
        # path. This means the RTEMS specific automake is not found and which
        # breaks the bootstrap. We need to remove the prepended path. Also
        # remove any ACLOCAL paths from the environment.
        #
        if os.name == 'nt':
            cspath = os.environ['PATH'].split(os.pathsep)
            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
                os.environ['PATH'] = os.pathsep.join(cspath[1:])

        top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
        prefix = '/opt/rtems/%s' % (rtems_version())
        tools = prefix
        build_dir = 'bsp-builds'
        logf = 'bsp-build-%s.txt' % (
            datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
        config_file = path.join(top, 'share', 'rtems', 'tester', 'rtems',
                                'rtems-bsps.ini')
        if not path.exists(config_file):
            config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')

        argsp = argparse.ArgumentParser()
        argsp.add_argument('--prefix',
                           help='Prefix to build the BSP.',
                           type=str)
        argsp.add_argument('--rtems-tools',
                           help='The RTEMS tools directory.',
                           type=str)
        argsp.add_argument('--rtems', help='The RTEMS source tree.', type=str)
        argsp.add_argument('--build-path', help='Path to build in.', type=str)
        argsp.add_argument('--log', help='Log file.', type=str)
        argsp.add_argument('--stop-on-error',
                           help='Stop on an error.',
                           action='store_true')
        argsp.add_argument('--no-clean',
                           help='Do not clean the build output.',
                           action='store_true')
        argsp.add_argument('--profiles',
                           help='Build the listed profiles.',
                           type=str,
                           default='tier-1')
        argsp.add_argument('--build', help='Build variation.', type=str)
        argsp.add_argument('--arch',
                           help='Build the specific architecture.',
                           type=str)
        argsp.add_argument('--bsp', help='Build the specific BSP.', type=str)
        argsp.add_argument('--dry-run',
                           help='Do not run the actual builds.',
                           action='store_true')

        opts = argsp.parse_args(args[1:])
        if opts.log is not None:
            logf = opts.log
        log.default = log.log([logf])
        log.notice('RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' %
                   (version.str()))
        if opts.rtems is None:
            raise error.general('No RTEMS source provided on the command line')
        if opts.prefix is not None:
            prefix = path.shell(opts.prefix)
        if opts.rtems_tools is not None:
            tools = path.shell(opts.rtems_tools)
        if opts.build_path is not None:
            build_dir = path.shell(opts.build_path)
        if opts.bsp is not None and opts.arch is None:
            raise error.general('BSP provided but no architecture')

        config = configuration()
        config.load(config_file, opts.build)

        options = {
            'stop-on-error': opts.stop_on_error,
            'no-clean': opts.no_clean,
            'dry-run': opts.dry_run,
            'jobs': 8
        }

        b = build(config, rtems_version(), prefix, tools,
                  path.shell(opts.rtems), build_dir, options)
        if opts.arch is not None:
            if opts.bsp is not None:
                b.build_arch_bsp(opts.arch, opts.bsp)
            else:
                b.build_arch(opts.arch)
        else:
            for profile in opts.profiles.split(','):
                b.build_profile(profile.strip())

    except error.general as gerr:
        print(gerr)
        print('BSP Build FAILED', file=sys.stderr)
        ec = 1
    except error.internal as ierr:
        print(ierr)
        print('BSP Build FAILED', file=sys.stderr)
        ec = 1
    except error.exit as eerr:
        pass
    except KeyboardInterrupt:
        log.notice('abort: user terminated')
        ec = 1
    if b is not None:
        b.results.report()
    sys.exit(ec)
Example #7
0
def run(command_path = None):
    import sys
    tests = []
    stdtty = console.save()
    opts = None
    default_exefilter = '*.exe'
    try:
        optargs = { '--rtems-tools': 'The path to the RTEMS tools',
                    '--rtems-bsp':   'The RTEMS BSP to run the test on',
                    '--user-config': 'Path to your local user configuration INI file',
                    '--list-bsps':   'List the supported BSPs',
                    '--debug-trace': 'Debug trace based on specific flags',
                    '--stacktrace':  'Dump a stack trace on a user termination (^C)' }
        opts = options.load(sys.argv,
                            optargs = optargs,
                            command_path = command_path)
        log.notice('RTEMS Testing - Run, %s' % (version.str()))
        if opts.find_arg('--list-bsps'):
            bsps.list(opts)
        opts.log_info()
        log.output('Host: ' + host.label(mode = 'all'))
        debug_trace = opts.find_arg('--debug-trace')
        if debug_trace:
            if len(debug_trace) != 1:
                debug_trace = 'output,' + debug_trace[1]
            else:
                raise error.general('no debug flags, can be: console,gdb,output')
        else:
            debug_trace = 'output'
        opts.defaults['debug_trace'] = debug_trace
        rtems_tools = opts.find_arg('--rtems-tools')
        if rtems_tools:
            if len(rtems_tools) != 2:
                raise error.general('invalid RTEMS tools option')
            rtems_tools = rtems_tools[1]
        else:
            rtems_tools = '%{_prefix}'
        bsp = opts.find_arg('--rtems-bsp')
        if bsp is None or len(bsp) != 2:
            raise error.general('RTEMS BSP not provided or an invalid option')
        bsp = config.load(bsp[1], opts)
        bsp_config = opts.defaults.expand(opts.defaults['tester'])
        executables = find_executables(opts.params())
        if len(executables) != 1:
            raise error.general('one executable required, found %d' % (len(executables)))
        opts.defaults['test_disable_header'] = '1'
        reports = report.report(1)
        start_time = datetime.datetime.now()
        opts.defaults['exe_trace'] = debug_trace
        tst = test(1, 1, reports, executables[0], rtems_tools, bsp, bsp_config, opts)
        tst.run()
        end_time = datetime.datetime.now()
        total_time = 'Run time     : %s' % (str(end_time - start_time))
        log.notice(total_time)

    except error.general as gerr:
        print(gerr)
        sys.exit(1)
    except error.internal as ierr:
        print(ierr)
        sys.exit(1)
    except error.exit:
        sys.exit(2)
    except KeyboardInterrupt:
        if opts is not None and opts.find_arg('--stacktrace'):
            print('}} dumping:', threading.active_count())
            for t in threading.enumerate():
                print('}} ', t.name)
            print(stacktraces.trace())
        log.notice('abort: user terminated')
        sys.exit(1)
    finally:
        console.restore(stdtty)
    sys.exit(0)
Example #8
0
def run(command_path = None):
    import sys
    tests = []
    stdtty = console.save()
    opts = None
    default_exefilter = '*.exe'
    try:
        optargs = { '--rtems-tools': 'The path to the RTEMS tools',
                    '--rtems-bsp':   'The RTEMS BSP to run the test on',
                    '--user-config': 'Path to your local user configuration INI file',
                    '--report-mode': 'Reporting modes, failures (default),all,none',
                    '--list-bsps':   'List the supported BSPs',
                    '--debug-trace': 'Debug trace based on specific flags',
                    '--filter':      'Glob that executables must match to run (default: ' +
                              default_exefilter + ')',
                    '--stacktrace':  'Dump a stack trace on a user termination (^C)' }
        mailer.append_options(optargs)
        opts = options.load(sys.argv,
                            optargs = optargs,
                            command_path = command_path)
        mail = None
        output = None
        if opts.find_arg('--mail'):
            mail = mailer.mail(opts)
            # Request these now to generate any errors.
            from_addr = mail.from_address()
            smtp_host = mail.smtp_host()
            to_addr = opts.find_arg('--mail-to')
            if to_addr:
                to_addr = to_addr[1]
            else:
                to_addr = '*****@*****.**'
            output = log_capture()
        log.notice('RTEMS Testing - Tester, %s' % (version.str()))
        if opts.find_arg('--list-bsps'):
            bsps.list(opts)
        exe_filter = opts.find_arg('--filter')
        if exe_filter:
            exe_filter = exe_filter[1]
        else:
            exe_filter = default_exefilter
        opts.log_info()
        log.output('Host: ' + host.label(mode = 'all'))
        debug_trace = opts.find_arg('--debug-trace')
        if debug_trace:
            if len(debug_trace) != 1:
                debug_trace = debug_trace[1]
            else:
                raise error.general('no debug flags, can be: console,gdb,output')
        else:
            debug_trace = ''
        opts.defaults['exe_trace'] = debug_trace
        job_trace = 'jobs' in debug_trace.split(',')
        rtems_tools = opts.find_arg('--rtems-tools')
        if rtems_tools:
            if len(rtems_tools) != 2:
                raise error.general('invalid RTEMS tools option')
            rtems_tools = rtems_tools[1]
        else:
            rtems_tools = '%{_prefix}'
        bsp = opts.find_arg('--rtems-bsp')
        if bsp is None or len(bsp) != 2:
            raise error.general('RTEMS BSP not provided or an invalid option')
        bsp = config.load(bsp[1], opts)
        bsp_config = opts.defaults.expand(opts.defaults['tester'])
        report_mode = opts.find_arg('--report-mode')
        if report_mode:
            if report_mode[1] != 'failures' and \
                    report_mode[1] != 'all' and \
                    report_mode[1] != 'none':
                raise error.general('invalid report mode')
            report_mode = report_mode[1]
        else:
            report_mode = 'failures'
        executables = find_executables(opts.params(), exe_filter)
        if len(executables) == 0:
            raise error.general('no executables supplied')
        start_time = datetime.datetime.now()
        total = len(executables)
        reports = report.report(total)
        reporting = 1
        jobs = int(opts.jobs(opts.defaults['_ncpus']))
        exe = 0
        finished = []
        if jobs > len(executables):
            jobs = len(executables)
        while exe < total or len(tests) > 0:
            if exe < total and len(tests) < jobs:
                tst = test_run(exe + 1, total, reports,
                               executables[exe],
                               rtems_tools, bsp, bsp_config,
                               opts)
                exe += 1
                tests += [tst]
                if job_trace:
                    _job_trace(tst, 'create',
                               total, exe, tests, reporting)
                tst.run()
            else:
                dead = [t for t in tests if not t.is_alive()]
                tests[:] = [t for t in tests if t not in dead]
                for tst in dead:
                    if job_trace:
                        _job_trace(tst, 'dead',
                                   total, exe, tests, reporting)
                    finished += [tst]
                    tst.reraise()
                del dead
                if len(tests) >= jobs or exe >= total:
                    time.sleep(0.250)
                if len(finished):
                    reporting = report_finished(reports,
                                                report_mode,
                                                reporting,
                                                finished,
                                                job_trace)
        finished_time = datetime.datetime.now()
        reporting = report_finished(reports, report_mode,
                                    reporting, finished, job_trace)
        if reporting < total:
            log.warning('finished jobs does match: %d' % (reporting))
            report_finished(reports, report_mode, -1, finished, job_trace)
        reports.summary()
        end_time = datetime.datetime.now()
        average_time = 'Average test time: %s' % (str((end_time - start_time) / total))
        total_time = 'Testing time     : %s' % (str(end_time - start_time))
        log.notice(average_time)
        log.notice(total_time)
        if mail is not None and output is not None:
            m_arch = opts.defaults.expand('%{arch}')
            m_bsp = opts.defaults.expand('%{bsp}')
            build = ' %s:' % (reports.get_config('build', not_found = ''))
            subject = '[rtems-test] %s/%s:%s %s' % (m_arch,
                                                    m_bsp,
                                                    build,
                                                    reports.score_card('short'))
            np = 'Not present in test'
            ver = reports.get_config('version', not_found = np)
            build = reports.get_config('build', not_found = np)
            tools = reports.get_config('tools', not_found = np)
            body = [total_time, average_time,
                    '', 'Host', '====', host.label(mode = 'all'),
                    '', 'Configuration', '=============',
                    'Version: %s' % (ver),
                    'Build  : %s' % (build),
                    'Tools  : %s' % (tools),
                    '', 'Summary', '=======', '',
                    reports.score_card(), '',
                    reports.failures(),
                    'Log', '===', ''] + output.get()
            mail.send(to_addr, subject, os.linesep.join(body))

    except error.general as gerr:
        print(gerr)
        sys.exit(1)
    except error.internal as ierr:
        print(ierr)
        sys.exit(1)
    except error.exit:
        sys.exit(2)
    except KeyboardInterrupt:
        if opts is not None and opts.find_arg('--stacktrace'):
            print('}} dumping:', threading.active_count())
            for t in threading.enumerate():
                print('}} ', t.name)
            print(stacktraces.trace())
        log.notice('abort: user terminated')
        killall(tests)
        sys.exit(1)
    finally:
        console.restore(stdtty)
    sys.exit(0)