コード例 #1
0
def do_printenv(m: Main, set_prog: bool = True) -> int:
    """Print the environment for the given spec."""
    if set_prog:
        m.argument_parser.prog = m.argument_parser.prog + " printenv"
    m.argument_parser.add_argument(
        "spec_name",
        help="spec to build. This is "
        "the basename of an .anod file (without the extension)",
    )
    m.argument_parser.add_argument("--qualifier", help="optional qualifier")
    m.argument_parser.add_argument(
        "--sandbox-dir",
        help="directory in which build artifacts are stored",
        default=SBX_DIR,
    )
    m.argument_parser.add_argument(
        "--build-env",
        help="print build environment",
        action="store_true",
        default=False,
    )
    m.parse_args()

    # Disable logging messages except errors
    logging.getLogger("").setLevel(logging.ERROR)

    check_common_tools()

    ac = create_anod_context(SPEC_DIR)
    sbx = create_anod_sandbox(m.args.sandbox_dir, SPEC_DIR)

    anod_instance = ac.add_anod_action(
        name=m.args.spec_name,
        primitive="build",
        qualifier=m.args.qualifier,
        sandbox=sbx,
        upload=False,
        env=BaseEnv.from_env(),
    ).anod_instance

    saved_env = {k: v for k, v in os.environ.items()}

    if m.args.build_env:
        if hasattr(anod_instance, "build_setenv"):
            anod_instance.build_setenv()
    else:
        if hasattr(anod_instance, "setenv"):
            anod_instance.setenv()

    for var, value in os.environ.items():
        if var not in saved_env or saved_env[var] != os.environ[var]:
            print('export %s="%s";' % (var, value))

            if m.args.verbose >= 1:
                print('printf "I set %s=\\"%s\\"\\n\\n";' % (var, value))

            print(" ")

    print(BANNER % m.args.spec_name)
    return 0
コード例 #2
0
def main():
    """Run e3-test script."""
    m = Main()

    # Ignore arguments here as they are arguments for the actual testsuite
    m.parse_args(known_args_only=True)

    # Find first the tool configuration file. Keep track of current directory
    # that will be used to select the test subset automatically.
    cwd = os.path.abspath(os.getcwd())
    root_dir = cwd
    while not os.path.isfile(os.path.join(root_dir, 'e3-test.yaml')):
        new_root_dir = os.path.dirname(root_dir)
        if new_root_dir == root_dir:
            logging.error("cannot find e3-test.yaml")
            return 1
        root_dir = new_root_dir
    config_file = os.path.join(root_dir, 'e3-test.yaml')

    with open(config_file, 'rb') as fd:
        config = yaml.load(fd)

    if 'main' not in config:
        logging.error('cannot find testsuite main')
        return 1
    p = Run([
        sys.executable,
        os.path.join(root_dir, config['main']),
        os.path.relpath(cwd, root_dir) + '/'
    ] + config.get('default_args', []),
            output=None,
            cwd=root_dir)
    return p.status
コード例 #3
0
ファイル: scripts.py プロジェクト: onedigitallife/e3-core
def anod():
    """bin/anod script entry point.

    This script is installed in the sandbox.
    """
    import os
    import sys

    import e3.anod.driver
    import e3.anod.loader
    import e3.anod.sandbox
    import e3.env
    import e3.store
    import e3.store.cache

    sandbox_dir = os.path.abspath(
        os.path.join(os.path.dirname(sys.modules['__main__'].__file__),
                     os.pardir))

    sandbox = e3.anod.sandbox.SandBox()
    sandbox.root_dir = sandbox_dir

    # Load the local specs
    spec_repo = e3.anod.loader.AnodSpecRepository(sandbox.specs_dir)

    # Load the cache
    cache = e3.store.cache.load_cache('file-cache',
                                      {'cache_dir': sandbox.tmp_cache_dir})

    store = e3.store.load_store('http-simple-store', {}, cache)

    m = Main()
    subparsers = m.argument_parser.add_subparsers()
    anod_cmdline(subparsers, 'download', 'download a binary package')
    m.parse_args()

    action = m.args.action_name
    spec = m.args.spec
    qualifier = m.args.qualifier

    anod_cls = spec_repo.load(name=spec)
    anod_instance = anod_cls(qualifier=qualifier,
                             kind=action,
                             jobs=1,
                             env=e3.env.BaseEnv.from_env())

    # ??? inject the sandbox
    anod_instance.sandbox = sandbox

    driver = e3.anod.driver.AnodDriver(anod_instance=anod_instance,
                                       store=store)

    try:
        driver.activate(sandbox, spec_repo)
        driver.call(action)
    except AnodError as err:
        print(err, file=sys.stderr)
        sys.exit(1)
コード例 #4
0
def do_build(m: Main, set_prog: bool = True) -> int:
    """Perform the build."""
    if set_prog:
        m.argument_parser.prog = m.argument_parser.prog + " build"
    m.argument_parser.add_argument(
        "spec_name",
        help="spec to build. This is "
        "the basename of an .anod file (without the extension)",
    )
    m.argument_parser.add_argument("--qualifier", help="optional qualifier")
    m.argument_parser.add_argument(
        "--sandbox-dir",
        help="directory in which build artefacts are stored",
        default=SBX_DIR,
    )
    m.argument_parser.add_argument(
        "--force",
        help="force rebuild of everything",
        action="store_true",
        default=False,
    )
    m.parse_args()

    check_common_tools()

    ac = create_anod_context(SPEC_DIR)
    sbx = create_anod_sandbox(m.args.sandbox_dir, SPEC_DIR)

    sbx.create_dirs()

    ac.add_anod_action(
        name=m.args.spec_name,
        primitive="build",
        qualifier=m.args.qualifier,
        sandbox=sbx,
        upload=False,
        env=BaseEnv.from_env(),
    )
    actions = ac.schedule(resolver=ac.always_create_source_resolver)

    walker = UxasBuilder(actions, sandbox=sbx, force=m.args.force)

    # TODO: something with walker.job_status['root'], assuming we can get a
    # useful value there. Right now, it's always 'unknown'
    #
    # In the meantime, python > 3.6 guarantees the order of keys in a dict.
    # The job_status dict has as its penultimate entry the thing we asked to
    # build or the last thing that failed (the last non-root node). It's ugly,
    # but _should_ be safe to use this, until we have resolution for root
    # always reporting unknown.
    result: ReturnValue = list(walker.job_status.values())[-2]

    if result in BUILD_SUCCESS:
        return 0
    else:
        return result.value
コード例 #5
0
def main(get_argument_parser=False):
    """Manipulate an Anod sandbox.

    This function creates the main code for the entry-point e3-sandbox. To
    create new actions it is possible to create new sandbox plugins. e.g. to
    add a new plugin ``foo`` from a package ``e3-contrib``, derives the
    class :class:`SandBoxAction` and register the extension by adding in
    :file:`e3-contrib/setup.py`::

        entry_points={
            'e3.anod.sandbox.sandbox_action': [
                'foo = e3_contrib.sandbox_actions.SandBoxFoo']
        }

    :param get_argument_parser: return e3.main.Main argument_parser instead
        of running the action.
    :type get_argument_parser: bool
    """
    m = Main()
    m.parse_args(known_args_only=True)

    subparsers = m.argument_parser.add_subparsers(title="action",
                                                  description="valid actions")

    # Load all sandbox actions plugins
    ext = stevedore.ExtensionManager(
        namespace="e3.anod.sandbox.sandbox_action",
        invoke_on_load=True,
        invoke_args=(subparsers, ),
    )

    if len(ext.names()) != len(ext.entry_points_names()):
        raise SandBoxError(
            "an error occured when loading sandbox_action entry points %s" %
            ",".join(ext.entry_points_names()))  # defensive code

    if get_argument_parser:
        return m.argument_parser

    args = m.argument_parser.parse_args()

    e3.log.debug("sandbox action plugins loaded: %s", ",".join(ext.names()))

    # An action has been selected, run it
    try:
        ext[args.action].obj.run(args)
    except SandBoxError as err:
        logger.error(err)
        sys.exit(1)
コード例 #6
0
ファイル: main.py プロジェクト: AdaCore/e3-core
def main(get_argument_parser=False):
    """Manipulate an Anod sandbox.

    This function creates the main code for the entry-point e3-sandbox. To
    create new actions it is possible to create new sandbox plugins. e.g. to
    add a new plugin ``foo`` from a package ``e3-contrib``, derives the
    class :class:`SandBoxAction` and register the extension by adding in
    :file:`e3-contrib/setup.py`::

        entry_points={
            'e3.anod.sandbox.sandbox_action': [
                'foo = e3_contrib.sandbox_actions.SandBoxFoo']
        }

    :param get_argument_parser: return e3.main.Main argument_parser instead
        of running the action.
    :type get_argument_parser: bool
    """
    m = Main()
    subparsers = m.argument_parser.add_subparsers(
        title="action", description="valid actions")

    # Load all sandbox actions plugins
    ext = stevedore.ExtensionManager(
        namespace='e3.anod.sandbox.sandbox_action',
        invoke_on_load=True,
        invoke_args=(subparsers, ))

    if get_argument_parser:
        return m.argument_parser

    m.parse_args()

    e3.log.debug('sandbox action plugins loaded: %s',
                 ','.join(ext.names()))

    # An action has been selected, run it
    ext[m.args.action].obj.run(m.args)
コード例 #7
0
ファイル: context.py プロジェクト: AdaCore/gnatcoverage
    def __cmdline_options(self):
        """Return an options object to represent the command line options"""
        main = Main(platform_args=True)
        parser = main.argument_parser
        parser.add_argument('--timeout', type=int, default=None)
        parser.add_argument('--report-file',
                            metavar='FILE',
                            help='The filename where to store the test report'
                            ' [required]')
        parser.add_argument('--qualif-level',
                            metavar='QUALIF_LEVEL',
                            help='The target qualification level when we are'
                            ' running in qualification mode.')

        parser.add_argument('--xcov-level',
                            help='Force the --level argument passed to xcov'
                            'instead of deducing it from the test'
                            ' category when that normally happens.')

        parser.add_argument('--tags', default="")

        control.add_shared_options_to(parser, toplevel=False)

        main.parse_args()

        # "--report-file" is a required "option" which is a bit
        # self-contradictory, but it's easy to do it that way.
        exit_if(main.args.report_file is None,
                "The report file must be specified with --report-file")

        # Get our tags set as a list. Fetch contents from file if needed
        # first:
        if main.args.tags and main.args.tags.startswith('@'):
            main.args.tags = ' '.join(lines_of(main.args.tags[1:]))
        if main.args.tags:
            main.args.tags = main.args.tags.split()

        return main.args
コード例 #8
0
def main() -> None:
    """Provide entry point."""
    parser = argparse.ArgumentParser(
        description="Launch command with AWS credentials")

    parser.add_argument("--region",
                        help="AWS region to use",
                        default="eu-west-1")
    parser.add_argument("--profile",
                        help="AWS profile to use to run the command.",
                        default=None)
    parser.add_argument(
        "--role-arn",
        help="ARN of the role to assume to run the command.",
        required=True,
    )
    parser.add_argument(
        "--session_duration",
        help="session duration in seconds or None for default",
        default=None,
    )
    parser.add_argument("command")

    main_parser = Main(argument_parser=parser)
    main_parser.parse_args()
    assert main_parser.args is not None

    session = Session(profile=main_parser.args.profile,
                      regions=[main_parser.args.region])

    session.run(
        main_parser.args.command.split(),
        role_arn=main_parser.args.role_arn,
        session_duration=main_parser.args.session_duration,
        output=None,
    )
コード例 #9
0
    ac.add_anod_action(
        name=m.args.spec_name,
        primitive="build",
        qualifier=m.args.qualifier,
        sandbox=sbx,
        upload=False,
        env=BaseEnv.from_env(),
    )
    actions = ac.schedule(resolver=ac.always_create_source_resolver)

    walker = UxasBuilder(actions, sandbox=sbx, force=m.args.force)

    # TODO: something with walker.job_status['root'], assuming we can get a
    # useful value there. Right now, it's always 'unknown'
    #
    # In the meantime, python > 3.6 guarantees the order of keys in a dict.
    # The job_status dict has as its penultimate entry the thing we asked to
    # build or the last thing that failed (the last non-root node). It's ugly,
    # but _should_ be safe to use this, until we have resolution for root
    # always reporting unknown.
    result: ReturnValue = list(walker.job_status.values())[-2]

    if result in BUILD_SUCCESS:
        return 0
    else:
        return result.value


if __name__ == "__main__":
    exit(do_build(Main(), set_prog=False))
コード例 #10
0
ファイル: __init__.py プロジェクト: pmderodat/e3-testsuite
    def testsuite_main(self, args: Optional[List[str]] = None) -> int:
        """Main for the main testsuite script.

        :param args: Command line arguments. If None, use `sys.argv`.
        :return: The testsuite status code (0 for success, a positive for
            failure).
        """
        self.main = Main(platform_args=True)

        # Add common options
        parser = self.main.argument_parser

        temp_group = parser.add_argument_group(
            title="temporaries handling arguments")
        temp_group.add_argument("-t",
                                "--temp-dir",
                                metavar="DIR",
                                default=Env().tmp_dir)
        temp_group.add_argument(
            "--no-random-temp-subdir",
            dest="random_temp_subdir",
            action="store_false",
            help="Disable the creation of a random subdirectory in the"
            " temporary directory. Use this when you know that you have"
            " exclusive access to the temporary directory (needed in order to"
            " avoid name clashes there) to get a deterministic path for"
            " testsuite temporaries.")
        temp_group.add_argument(
            "-d",
            "--dev-temp",
            metavar="DIR",
            nargs="?",
            default=None,
            const="tmp",
            help="Convenience shortcut for dev setups: forces `-t DIR"
            " --no-random-temp-subdir --cleanup-mode=none` and cleans up `DIR`"
            ' first. If no directory is provided, use the local "tmp"'
            " directory.")

        cleanup_mode_map = enum_to_cmdline_args_map(CleanupMode)
        temp_group.add_argument(
            "--cleanup-mode",
            choices=list(cleanup_mode_map),
            help="Control the cleanup of working spaces.\n" +
            "\n".join(f"{name}: {CleanupMode.descriptions()[value]}"
                      for name, value in cleanup_mode_map.items()))
        temp_group.add_argument(
            "--disable-cleanup",
            action="store_true",
            help="Disable cleanup of working spaces. This option is deprecated"
            " and will disappear in a future version of e3-testsuite. Please"
            " use --cleanup-mode instead.")

        output_group = parser.add_argument_group(
            title="results output arguments")
        output_group.add_argument(
            "-o",
            "--output-dir",
            metavar="DIR",
            default="./out",
            help="Select the output directory, where test results are to be"
            " stored (default: './out'). If --old-output-dir=DIR2 is passed,"
            " the new results are stored in DIR while DIR2 contains results"
            " from a previous run. Otherwise, the new results are stored in"
            " DIR/new/ while the old ones are stored in DIR/old. In both"
            " cases, the testsuite cleans the directory for new results"
            " first.",
        )
        output_group.add_argument(
            "--old-output-dir",
            metavar="DIR",
            help="Select the old output directory, for baseline comparison."
            " See --output-dir.",
        )
        output_group.add_argument(
            "--rotate-output-dirs",
            default=False,
            action="store_true",
            help="Rotate testsuite results: move the new results directory to"
            " the old results one before running testcases (this removes the"
            " old results directory first). If not passed, we just remove the"
            " new results directory before running testcases (i.e. just ignore"
            " the old results directory).",
        )
        output_group.add_argument(
            "--show-error-output",
            "-E",
            action="store_true",
            help="When testcases fail, display their output. This is for"
            " convenience for interactive use.",
        )
        output_group.add_argument(
            "--show-time-info",
            action="store_true",
            help="Display time information for test results, if available",
        )
        output_group.add_argument(
            "--xunit-output",
            dest="xunit_output",
            metavar="FILE",
            help="Output testsuite report to the given file in the standard"
            " XUnit XML format. This is useful to display results in"
            " continuous build systems such as Jenkins.",
        )
        output_group.add_argument(
            "--gaia-output",
            action="store_true",
            help="Output a GAIA-compatible testsuite report next to the YAML"
            " report.",
        )
        output_group.add_argument(
            "--status-update-interval",
            default=1.0,
            type=float,
            help="Minimum number of seconds between status file updates. The"
            " more often we update this file, the more often one will read"
            " garbage.")

        auto_gen_default = ("enabled"
                            if self.auto_generate_text_report else "disabled")
        output_group.add_argument(
            "--generate-text-report",
            action="store_true",
            dest="generate_text_report",
            default=self.auto_generate_text_report,
            help=(
                f"When the testsuite completes, generate a 'report' text file"
                f" in the output directory ({auto_gen_default} by default)."),
        )
        output_group.add_argument(
            "--no-generate-text-report",
            action="store_false",
            dest="generate_text_report",
            help="Disable the generation of a 'report' text file (see"
            "--generate-text-report).",
        )

        output_group.add_argument(
            "--truncate-logs",
            "-T",
            metavar="N",
            type=int,
            default=200,
            help="When outputs (for instance subprocess outputs) exceed 2*N"
            " lines, only include the first and last N lines in logs. This is"
            " necessary when storage for testsuite results have size limits,"
            " and the useful information is generally either at the beginning"
            " or the end of such outputs. If 0, never truncate logs.",
        )
        output_group.add_argument(
            "--dump-environ",
            dest="dump_environ",
            action="store_true",
            default=False,
            help="Dump all environment variables in a file named environ.sh,"
            " located in the output directory (see --output-dir). This"
            " file can then be sourced from a Bourne shell to recreate"
            " the environement that existed when this testsuite was run"
            " to produce a given testsuite report.",
        )

        exec_group = parser.add_argument_group(
            title="execution control arguments")
        exec_group.add_argument(
            "--max-consecutive-failures",
            "-M",
            metavar="N",
            type=int,
            default=self.default_max_consecutive_failures,
            help="Number of test failures (FAIL or ERROR) that trigger the"
            " abortion of the testuite. If zero, this behavior is disabled. In"
            " some cases, aborting the testsuite when there are just too many"
            " failures saves time and costs: the software to test/environment"
            " is too broken, there is no point to continue running the"
            " testsuite.",
        )
        exec_group.add_argument(
            "-j",
            "--jobs",
            dest="jobs",
            type=int,
            metavar="N",
            default=Env().build.cpu.cores,
            help="Specify the number of jobs to run simultaneously",
        )
        exec_group.add_argument(
            "--failure-exit-code",
            metavar="N",
            type=int,
            default=self.default_failure_exit_code,
            help="Exit code the testsuite must use when at least one test"
            " result shows a failure/error. By default, this is"
            f" {self.default_failure_exit_code}. This option is useful when"
            " running a testsuite in a continuous integration setup, as this"
            " can make the testing process stop when there is a regression.",
        )
        exec_group.add_argument(
            "--force-multiprocessing",
            action="store_true",
            help="Force the use of subprocesses to execute tests, for"
            " debugging purposes. This is normally automatically enabled when"
            " both the level of requested parallelism is high enough (to make"
            " it profitable regarding the contention of Python's GIL) and no"
            " test fragment has dependencies on other fragments. This flag"
            " forces the use of multiprocessing even if any of these two"
            " conditions is false.")
        parser.add_argument("sublist",
                            metavar="tests",
                            nargs="*",
                            default=[],
                            help="test")
        # Add user defined options
        self.add_options(parser)

        # Parse options
        self.main.parse_args(args)
        assert self.main.args is not None

        # If there is a chance for the logging to end up in a non-tty stream,
        # disable colors. If not, be user-friendly and automatically show error
        # outputs.
        if (self.main.args.log_file or not isatty(sys.stdout)
                or not isatty(sys.stderr)):
            enable_colors = False
        else:  # interactive-only
            enable_colors = True
            self.main.args.show_error_output = True
        self.colors = ColorConfig(enable_colors)
        self.Fore = self.colors.Fore
        self.Style = self.colors.Style

        self.env = Env()
        self.env.enable_colors = enable_colors
        self.env.root_dir = self.root_dir
        self.env.test_dir = self.test_dir

        # Setup output directories and create an index for the results we are
        # going to produce.
        self.output_dir: str
        self.old_output_dir: Optional[str]
        self.setup_result_dirs()
        self.report_index = ReportIndex(self.output_dir)

        # Set the cleanup mode from command-line arguments
        if self.main.args.cleanup_mode is not None:
            self.env.cleanup_mode = (
                cleanup_mode_map[self.main.args.cleanup_mode])
        elif self.main.args.disable_cleanup:
            logger.warning(
                "--disable-cleanup is deprecated and will disappear in a"
                " future version of e3-testsuite. Please use --cleanup-mode"
                " instead.")
            self.env.cleanup_mode = CleanupMode.NONE
        else:
            self.env.cleanup_mode = CleanupMode.default()

        # Settings for temporary directory creation
        temp_dir: str = self.main.args.temp_dir
        random_temp_subdir: bool = self.main.args.random_temp_subdir

        # The "--dev-temp" option forces several settings
        if self.main.args.dev_temp:
            self.env.cleanup_mode = CleanupMode.NONE
            temp_dir = self.main.args.dev_temp
            random_temp_subdir = False

        # Now actually setup the temporary directory: make sure we start from a
        # clean directory if we use a deterministic directory.
        #
        # Note that we do make sure that working_dir is an absolute path, as we
        # are likely to be changing directories when running each test. A
        # relative path would no longer work under those circumstances.
        temp_dir = os.path.abspath(temp_dir)
        if not random_temp_subdir:
            self.working_dir = temp_dir
            rm(self.working_dir, recursive=True)
            mkdir(self.working_dir)

        elif not os.path.isdir(temp_dir):
            # If the temp dir is supposed to be randomized, we need to create a
            # subdirectory, so check that the parent directory exists first.
            logger.critical("temp dir '%s' does not exist", temp_dir)
            return 1

        else:
            self.working_dir = tempfile.mkdtemp("", "tmp", temp_dir)

        # Create the exchange directory (to exchange data between the testsuite
        # main and the subprocesses running test fragments). Compute the name
        # of the file to pass environment data to subprocesses.
        self.exchange_dir = os.path.join(self.working_dir, "exchange")
        self.env_filename = os.path.join(self.exchange_dir, "_env.bin")
        mkdir(self.exchange_dir)

        # Make them both available to test fragments
        self.env.exchange_dir = self.exchange_dir
        self.env.env_filename = self.env_filename

        self.gaia_result_files: Dict[str, GAIAResultFiles] = {}
        """Mapping from test names to files for results in the GAIA report."""

        # Store in global env: target information and common paths
        self.env.output_dir = self.output_dir
        self.env.working_dir = self.working_dir
        self.env.options = self.main.args

        # Create an object to report testsuite execution status to users
        from e3.testsuite.running_status import RunningStatus
        self.running_status = RunningStatus(
            os.path.join(self.output_dir, "status"),
            self.main.args.status_update_interval,
        )

        # User specific startup
        self.set_up()

        # Retrieve the list of test
        self.test_list = self.get_test_list(self.main.args.sublist)

        # Create a DAG to constraint the test execution order
        dag = DAG()
        for parsed_test in self.test_list:
            self.add_test(dag, parsed_test)
        self.adjust_dag_dependencies(dag)
        dag.check()
        self.running_status.set_dag(dag)

        # Determine whether to use multiple processes for fragment execution
        # parallelism.
        self.use_multiprocessing = self.compute_use_multiprocessing()
        self.env.use_multiprocessing = self.use_multiprocessing

        # Record modules lookup path, including for the file corresponding to
        # the __main__ module.  Subprocesses will need it to have access to the
        # same modules.
        main_module = sys.modules["__main__"]
        self.env.modules_search_path = [
            os.path.dirname(os.path.abspath(main_module.__file__))
        ] + sys.path

        # Now that the env is supposed to be complete, dump it for the test
        # fragments to pick it up.
        self.env.store(self.env_filename)

        # For debugging purposes, dump the final DAG to a DOT file
        with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
            fd.write(dag.as_dot())

        if self.use_multiprocessing:
            self.run_multiprocess_mainloop(dag)
        else:
            self.run_standard_mainloop(dag)

        self.report_index.write()
        self.dump_testsuite_result()
        if self.main.args.xunit_output:
            dump_xunit_report(self, self.main.args.xunit_output)
        if self.main.args.gaia_output:
            dump_gaia_report(self, self.output_dir, self.gaia_result_files)

        # Clean everything
        self.tear_down()

        # If requested, generate a text report
        if self.main.args.generate_text_report:
            # Use the previous testsuite results for comparison, if available
            old_index = (ReportIndex.read(self.old_output_dir)
                         if self.old_output_dir else None)

            # Include all information, except logs for successful tests, which
            # is just too verbose.
            with open(os.path.join(self.output_dir, "report"),
                      "w",
                      encoding="utf-8") as f:
                generate_report(
                    output_file=f,
                    new_index=self.report_index,
                    old_index=old_index,
                    colors=ColorConfig(colors_enabled=False),
                    show_all_logs=False,
                    show_xfail_logs=True,
                    show_error_output=True,
                    show_time_info=True,
                )

        # Return the appropriate status code: 1 when there is a framework
        # issue, the failure status code from the --failure-exit-code=N option
        # when there is a least one testcase failure, or 0.
        statuses = {
            s
            for s, count in self.report_index.status_counters.items() if count
        }
        if TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
            return self.main.args.failure_exit_code
        else:
            return 0
コード例 #11
0
ファイル: __init__.py プロジェクト: pmderodat/e3-testsuite
class TestsuiteCore:
    """Testsuite Core driver.

    This class is the base of Testsuite class and should not be instanciated.
    It's not recommended to override any of the functions declared in it.

    See documentation of Testsuite class for overridable methods and
    variables.
    """
    def __init__(
        self,
        root_dir: Optional[str] = None,
        testsuite_name: str = "Untitled testsute",
    ) -> None:
        """Testsuite constructor.

        :param root_dir: Root directory for the testsuite. If left to None, use
            the directory containing the Python module that created self's
            class.
        :param testsuite_name: Name for this testsuite. It can be used to
            provide a title in some report formats.
        """
        if root_dir is None:
            root_dir = os.path.dirname(inspect.getfile(type(self)))
        self.root_dir = os.path.abspath(root_dir)
        self.test_dir = os.path.join(self.root_dir, self.tests_subdir)
        logger.debug("Test directory: %s", self.test_dir)
        self.consecutive_failures = 0
        self.return_values: Dict[str, Any] = {}
        self.result_tracebacks: Dict[str, List[str]] = {}
        self.testsuite_name = testsuite_name

        self.aborted_too_many_failures = False
        """
        Whether the testsuite aborted because of too many consecutive test
        failures (see the --max-consecutive-failures command-line option).
        """

        self.use_multiprocessing = False
        """Whether to use multi-processing for tests parallelism.

        Beyond a certain level of parallelism, Python's GIL contention is too
        high to benefit from more processors. When we reach this level, it is
        more interesting to use multiple processes to cancel the GIL
        contention.

        The actual value for this attribute is computed once the DAG is built,
        in the "compute_use_multiprocessing" method.
        """

    # Mypy does not support decorators on properties, so keep the actual
    # implementations for deprecated properties in methods.

    @deprecated(2)
    def _test_counter(self) -> int:
        return len(self.report_index.entries)

    @deprecated(2)
    def _test_status_counters(self) -> Dict[TestStatus, int]:
        return self.report_index.status_counters

    @deprecated(2)
    def _results(self) -> Dict[str, TestStatus]:
        return {
            e.test_name: e.status
            for e in self.report_index.entries.values()
        }

    @property
    def test_counter(self) -> int:
        """Return the number of test results in the report.

        Warning: this method is obsolete and will be removed in the future.
        """
        return self._test_counter()

    @property
    def test_status_counters(self) -> Dict[TestStatus, int]:
        """Return test result counts per test status.

        Warning: this method is obsolete and will be removed in the future.
        """
        return self._test_status_counters()

    @property
    def results(self) -> Dict[str, TestStatus]:
        """Return a mapping from test names to results.

        Warning: this method is obsolete and will be removed in the future.
        """
        return self._results()

    def compute_use_multiprocessing(self) -> bool:
        """Return whether to use multi-processing for tests parallelism.

        See docstring for the "use_multiprocessing" attribute. Subclasses are
        free to override this to take control of when multiprocessing is
        enabled. Note that this will disregard the "--force-multiprocessing"
        command line option.
        """
        raise NotImplementedError

    def testsuite_main(self, args: Optional[List[str]] = None) -> int:
        """Main for the main testsuite script.

        :param args: Command line arguments. If None, use `sys.argv`.
        :return: The testsuite status code (0 for success, a positive for
            failure).
        """
        self.main = Main(platform_args=True)

        # Add common options
        parser = self.main.argument_parser

        temp_group = parser.add_argument_group(
            title="temporaries handling arguments")
        temp_group.add_argument("-t",
                                "--temp-dir",
                                metavar="DIR",
                                default=Env().tmp_dir)
        temp_group.add_argument(
            "--no-random-temp-subdir",
            dest="random_temp_subdir",
            action="store_false",
            help="Disable the creation of a random subdirectory in the"
            " temporary directory. Use this when you know that you have"
            " exclusive access to the temporary directory (needed in order to"
            " avoid name clashes there) to get a deterministic path for"
            " testsuite temporaries.")
        temp_group.add_argument(
            "-d",
            "--dev-temp",
            metavar="DIR",
            nargs="?",
            default=None,
            const="tmp",
            help="Convenience shortcut for dev setups: forces `-t DIR"
            " --no-random-temp-subdir --cleanup-mode=none` and cleans up `DIR`"
            ' first. If no directory is provided, use the local "tmp"'
            " directory.")

        cleanup_mode_map = enum_to_cmdline_args_map(CleanupMode)
        temp_group.add_argument(
            "--cleanup-mode",
            choices=list(cleanup_mode_map),
            help="Control the cleanup of working spaces.\n" +
            "\n".join(f"{name}: {CleanupMode.descriptions()[value]}"
                      for name, value in cleanup_mode_map.items()))
        temp_group.add_argument(
            "--disable-cleanup",
            action="store_true",
            help="Disable cleanup of working spaces. This option is deprecated"
            " and will disappear in a future version of e3-testsuite. Please"
            " use --cleanup-mode instead.")

        output_group = parser.add_argument_group(
            title="results output arguments")
        output_group.add_argument(
            "-o",
            "--output-dir",
            metavar="DIR",
            default="./out",
            help="Select the output directory, where test results are to be"
            " stored (default: './out'). If --old-output-dir=DIR2 is passed,"
            " the new results are stored in DIR while DIR2 contains results"
            " from a previous run. Otherwise, the new results are stored in"
            " DIR/new/ while the old ones are stored in DIR/old. In both"
            " cases, the testsuite cleans the directory for new results"
            " first.",
        )
        output_group.add_argument(
            "--old-output-dir",
            metavar="DIR",
            help="Select the old output directory, for baseline comparison."
            " See --output-dir.",
        )
        output_group.add_argument(
            "--rotate-output-dirs",
            default=False,
            action="store_true",
            help="Rotate testsuite results: move the new results directory to"
            " the old results one before running testcases (this removes the"
            " old results directory first). If not passed, we just remove the"
            " new results directory before running testcases (i.e. just ignore"
            " the old results directory).",
        )
        output_group.add_argument(
            "--show-error-output",
            "-E",
            action="store_true",
            help="When testcases fail, display their output. This is for"
            " convenience for interactive use.",
        )
        output_group.add_argument(
            "--show-time-info",
            action="store_true",
            help="Display time information for test results, if available",
        )
        output_group.add_argument(
            "--xunit-output",
            dest="xunit_output",
            metavar="FILE",
            help="Output testsuite report to the given file in the standard"
            " XUnit XML format. This is useful to display results in"
            " continuous build systems such as Jenkins.",
        )
        output_group.add_argument(
            "--gaia-output",
            action="store_true",
            help="Output a GAIA-compatible testsuite report next to the YAML"
            " report.",
        )
        output_group.add_argument(
            "--status-update-interval",
            default=1.0,
            type=float,
            help="Minimum number of seconds between status file updates. The"
            " more often we update this file, the more often one will read"
            " garbage.")

        auto_gen_default = ("enabled"
                            if self.auto_generate_text_report else "disabled")
        output_group.add_argument(
            "--generate-text-report",
            action="store_true",
            dest="generate_text_report",
            default=self.auto_generate_text_report,
            help=(
                f"When the testsuite completes, generate a 'report' text file"
                f" in the output directory ({auto_gen_default} by default)."),
        )
        output_group.add_argument(
            "--no-generate-text-report",
            action="store_false",
            dest="generate_text_report",
            help="Disable the generation of a 'report' text file (see"
            "--generate-text-report).",
        )

        output_group.add_argument(
            "--truncate-logs",
            "-T",
            metavar="N",
            type=int,
            default=200,
            help="When outputs (for instance subprocess outputs) exceed 2*N"
            " lines, only include the first and last N lines in logs. This is"
            " necessary when storage for testsuite results have size limits,"
            " and the useful information is generally either at the beginning"
            " or the end of such outputs. If 0, never truncate logs.",
        )
        output_group.add_argument(
            "--dump-environ",
            dest="dump_environ",
            action="store_true",
            default=False,
            help="Dump all environment variables in a file named environ.sh,"
            " located in the output directory (see --output-dir). This"
            " file can then be sourced from a Bourne shell to recreate"
            " the environement that existed when this testsuite was run"
            " to produce a given testsuite report.",
        )

        exec_group = parser.add_argument_group(
            title="execution control arguments")
        exec_group.add_argument(
            "--max-consecutive-failures",
            "-M",
            metavar="N",
            type=int,
            default=self.default_max_consecutive_failures,
            help="Number of test failures (FAIL or ERROR) that trigger the"
            " abortion of the testuite. If zero, this behavior is disabled. In"
            " some cases, aborting the testsuite when there are just too many"
            " failures saves time and costs: the software to test/environment"
            " is too broken, there is no point to continue running the"
            " testsuite.",
        )
        exec_group.add_argument(
            "-j",
            "--jobs",
            dest="jobs",
            type=int,
            metavar="N",
            default=Env().build.cpu.cores,
            help="Specify the number of jobs to run simultaneously",
        )
        exec_group.add_argument(
            "--failure-exit-code",
            metavar="N",
            type=int,
            default=self.default_failure_exit_code,
            help="Exit code the testsuite must use when at least one test"
            " result shows a failure/error. By default, this is"
            f" {self.default_failure_exit_code}. This option is useful when"
            " running a testsuite in a continuous integration setup, as this"
            " can make the testing process stop when there is a regression.",
        )
        exec_group.add_argument(
            "--force-multiprocessing",
            action="store_true",
            help="Force the use of subprocesses to execute tests, for"
            " debugging purposes. This is normally automatically enabled when"
            " both the level of requested parallelism is high enough (to make"
            " it profitable regarding the contention of Python's GIL) and no"
            " test fragment has dependencies on other fragments. This flag"
            " forces the use of multiprocessing even if any of these two"
            " conditions is false.")
        parser.add_argument("sublist",
                            metavar="tests",
                            nargs="*",
                            default=[],
                            help="test")
        # Add user defined options
        self.add_options(parser)

        # Parse options
        self.main.parse_args(args)
        assert self.main.args is not None

        # If there is a chance for the logging to end up in a non-tty stream,
        # disable colors. If not, be user-friendly and automatically show error
        # outputs.
        if (self.main.args.log_file or not isatty(sys.stdout)
                or not isatty(sys.stderr)):
            enable_colors = False
        else:  # interactive-only
            enable_colors = True
            self.main.args.show_error_output = True
        self.colors = ColorConfig(enable_colors)
        self.Fore = self.colors.Fore
        self.Style = self.colors.Style

        self.env = Env()
        self.env.enable_colors = enable_colors
        self.env.root_dir = self.root_dir
        self.env.test_dir = self.test_dir

        # Setup output directories and create an index for the results we are
        # going to produce.
        self.output_dir: str
        self.old_output_dir: Optional[str]
        self.setup_result_dirs()
        self.report_index = ReportIndex(self.output_dir)

        # Set the cleanup mode from command-line arguments
        if self.main.args.cleanup_mode is not None:
            self.env.cleanup_mode = (
                cleanup_mode_map[self.main.args.cleanup_mode])
        elif self.main.args.disable_cleanup:
            logger.warning(
                "--disable-cleanup is deprecated and will disappear in a"
                " future version of e3-testsuite. Please use --cleanup-mode"
                " instead.")
            self.env.cleanup_mode = CleanupMode.NONE
        else:
            self.env.cleanup_mode = CleanupMode.default()

        # Settings for temporary directory creation
        temp_dir: str = self.main.args.temp_dir
        random_temp_subdir: bool = self.main.args.random_temp_subdir

        # The "--dev-temp" option forces several settings
        if self.main.args.dev_temp:
            self.env.cleanup_mode = CleanupMode.NONE
            temp_dir = self.main.args.dev_temp
            random_temp_subdir = False

        # Now actually setup the temporary directory: make sure we start from a
        # clean directory if we use a deterministic directory.
        #
        # Note that we do make sure that working_dir is an absolute path, as we
        # are likely to be changing directories when running each test. A
        # relative path would no longer work under those circumstances.
        temp_dir = os.path.abspath(temp_dir)
        if not random_temp_subdir:
            self.working_dir = temp_dir
            rm(self.working_dir, recursive=True)
            mkdir(self.working_dir)

        elif not os.path.isdir(temp_dir):
            # If the temp dir is supposed to be randomized, we need to create a
            # subdirectory, so check that the parent directory exists first.
            logger.critical("temp dir '%s' does not exist", temp_dir)
            return 1

        else:
            self.working_dir = tempfile.mkdtemp("", "tmp", temp_dir)

        # Create the exchange directory (to exchange data between the testsuite
        # main and the subprocesses running test fragments). Compute the name
        # of the file to pass environment data to subprocesses.
        self.exchange_dir = os.path.join(self.working_dir, "exchange")
        self.env_filename = os.path.join(self.exchange_dir, "_env.bin")
        mkdir(self.exchange_dir)

        # Make them both available to test fragments
        self.env.exchange_dir = self.exchange_dir
        self.env.env_filename = self.env_filename

        self.gaia_result_files: Dict[str, GAIAResultFiles] = {}
        """Mapping from test names to files for results in the GAIA report."""

        # Store in global env: target information and common paths
        self.env.output_dir = self.output_dir
        self.env.working_dir = self.working_dir
        self.env.options = self.main.args

        # Create an object to report testsuite execution status to users
        from e3.testsuite.running_status import RunningStatus
        self.running_status = RunningStatus(
            os.path.join(self.output_dir, "status"),
            self.main.args.status_update_interval,
        )

        # User specific startup
        self.set_up()

        # Retrieve the list of test
        self.test_list = self.get_test_list(self.main.args.sublist)

        # Create a DAG to constraint the test execution order
        dag = DAG()
        for parsed_test in self.test_list:
            self.add_test(dag, parsed_test)
        self.adjust_dag_dependencies(dag)
        dag.check()
        self.running_status.set_dag(dag)

        # Determine whether to use multiple processes for fragment execution
        # parallelism.
        self.use_multiprocessing = self.compute_use_multiprocessing()
        self.env.use_multiprocessing = self.use_multiprocessing

        # Record modules lookup path, including for the file corresponding to
        # the __main__ module.  Subprocesses will need it to have access to the
        # same modules.
        main_module = sys.modules["__main__"]
        self.env.modules_search_path = [
            os.path.dirname(os.path.abspath(main_module.__file__))
        ] + sys.path

        # Now that the env is supposed to be complete, dump it for the test
        # fragments to pick it up.
        self.env.store(self.env_filename)

        # For debugging purposes, dump the final DAG to a DOT file
        with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
            fd.write(dag.as_dot())

        if self.use_multiprocessing:
            self.run_multiprocess_mainloop(dag)
        else:
            self.run_standard_mainloop(dag)

        self.report_index.write()
        self.dump_testsuite_result()
        if self.main.args.xunit_output:
            dump_xunit_report(self, self.main.args.xunit_output)
        if self.main.args.gaia_output:
            dump_gaia_report(self, self.output_dir, self.gaia_result_files)

        # Clean everything
        self.tear_down()

        # If requested, generate a text report
        if self.main.args.generate_text_report:
            # Use the previous testsuite results for comparison, if available
            old_index = (ReportIndex.read(self.old_output_dir)
                         if self.old_output_dir else None)

            # Include all information, except logs for successful tests, which
            # is just too verbose.
            with open(os.path.join(self.output_dir, "report"),
                      "w",
                      encoding="utf-8") as f:
                generate_report(
                    output_file=f,
                    new_index=self.report_index,
                    old_index=old_index,
                    colors=ColorConfig(colors_enabled=False),
                    show_all_logs=False,
                    show_xfail_logs=True,
                    show_error_output=True,
                    show_time_info=True,
                )

        # Return the appropriate status code: 1 when there is a framework
        # issue, the failure status code from the --failure-exit-code=N option
        # when there is a least one testcase failure, or 0.
        statuses = {
            s
            for s, count in self.report_index.status_counters.items() if count
        }
        if TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
            return self.main.args.failure_exit_code
        else:
            return 0

    def get_test_list(self, sublist: List[str]) -> List[ParsedTest]:
        """Retrieve the list of tests.

        :param sublist: A list of tests scenarios or patterns.
        """
        # Use a mapping: test name -> ParsedTest when building the result, as
        # several patterns in "sublist" may yield the same testcase.
        testcases: Dict[str, ParsedTest] = {}
        test_finders = self.test_finders
        dedicated_dirs_only = all(tf.test_dedicated_directory
                                  for tf in test_finders)

        def matches_pattern(pattern: Optional[Pattern[str]],
                            name: str) -> bool:
            return pattern is None or bool(pattern.search(name))

        def add_testcase(pattern: Optional[Pattern[str]],
                         test: ParsedTest) -> None:

            # Do not add this testcase if its test-specific matcher does not
            # match the requested pattern.
            if test.test_matcher and not matches_pattern(
                    pattern, test.test_matcher):
                return

            if test.test_name in testcases:
                self.add_test_error(
                    test_name=test.test_name,
                    message=f"duplicate test name: {test.test_name}",
                    tb=None,
                )
            else:
                testcases[test.test_name] = test

        def helper(spec: str) -> None:
            pattern: Optional[Pattern[str]] = None

            # If the given pattern is a directory, do not go through the whole
            # tests subdirectory.
            if os.path.isdir(spec):
                root = spec
            else:
                root = self.test_dir
                try:
                    pattern = re.compile(spec)
                except re.error as exc:
                    logger.debug(
                        "Test pattern is not a valid regexp, try to match it"
                        " as-is: {}".format(exc))
                    pattern = re.compile(re.escape(spec))

            # For each directory in the requested subdir, ask our test finders
            # to probe for a testcase. Register matches.
            for dirpath, dirnames, filenames in os.walk(root,
                                                        followlinks=True):
                # If all tests are guaranteed to have a dedicated directory,
                # do not process directories that don't match the requested
                # pattern.
                if dedicated_dirs_only and not matches_pattern(
                        pattern, dirpath):
                    continue

                # The first test finder that has a match "wins". When handling
                # test data, we want to deal only with absolute paths, so get
                # the absolute name now.
                dirpath = os.path.abspath(dirpath)
                for tf in test_finders:
                    try:
                        test_or_list = tf.probe(self, dirpath, dirnames,
                                                filenames)
                    except ProbingError as exc:
                        self.add_test_error(
                            test_name=self.test_name(dirpath),
                            message=str(exc),
                            tb=traceback.format_exc(),
                        )
                        break
                    if isinstance(test_or_list, list):
                        for t in test_or_list:
                            add_testcase(pattern, t)
                        break
                    elif test_or_list is not None:
                        add_testcase(pattern, test_or_list)
                        break

        # If specific tests are requested, only look for them. Otherwise, just
        # look in the tests subdirectory.
        if sublist:
            for s in sublist:
                helper(s)
        else:
            helper(self.test_dir)

        result = list(testcases.values())
        logger.info("Found {} tests".format(len(result)))
        logger.debug("tests:\n  " + "\n  ".join(t.test_dir for t in result))
        return result

    def add_test(self, dag: DAG, parsed_test: ParsedTest) -> None:
        """Register a test to run.

        :param dag: The DAG of test fragments to execute for the testsuite.
        :param parsed_test: Test to instantiate.
        """
        test_name = parsed_test.test_name

        # Complete the test environment
        test_env = dict(parsed_test.test_env)
        test_env["test_dir"] = parsed_test.test_dir
        test_env["test_name"] = test_name

        assert isinstance(self.env.working_dir, str)
        test_env["working_dir"] = os.path.join(self.env.working_dir, test_name)

        # Fetch the test driver to use
        driver = parsed_test.driver_cls
        if not driver:
            if self.default_driver:
                driver = self.test_driver_map[self.default_driver]
            else:
                self.add_test_error(
                    test_name=test_name,
                    message="missing test driver",
                )
                return

        # Finally run the driver instantiation
        try:
            instance = driver(self.env, test_env)
            instance.Fore = self.Fore
            instance.Style = self.Style
            instance.add_test(dag)

        except Exception as e:
            self.add_test_error(
                test_name=test_name,
                message=str(e),
                tb=traceback.format_exc(),
            )

    def dump_testsuite_result(self) -> None:
        """Log a summary of test results.

        Subclasses are free to override this to do whatever is suitable for
        them.
        """
        lines = ["Summary:"]

        # Display test count for each status, but only for status that have
        # at least one test. Sort them by status value, to get consistent
        # order.
        def sort_key(couple: Tuple[TestStatus, int]) -> Any:
            status, _ = couple
            return status.value

        stats = sorted(
            ((status, count)
             for status, count in self.report_index.status_counters.items()
             if count),
            key=sort_key,
        )
        for status, count in stats:
            lines.append("  {}{: <12}{} {}".format(
                status.color(self.colors),
                status.name,
                self.Style.RESET_ALL,
                count,
            ))
        if not stats:
            lines.append("  <no test result>")
        logger.info("\n".join(lines))

        # Dump the comment file
        with open(os.path.join(self.output_dir, "comment"), "w") as f:
            self.write_comment_file(f)

    def collect_result(self, fragment: TestFragment) -> None:
        """Import test results from ``fragment`` into testsuite reports.

        :param fragment: Test fragment (just completed) from which to import
            test results.
        """
        assert self.main.args

        # Keep track of the number of consecutive failures seen so far if it
        # reaches the maximum number allowed, we must abort the testsuite.
        max_consecutive_failures = self.main.args.max_consecutive_failures

        while fragment.result_queue:
            item = fragment.result_queue.pop()

            self.add_result(item)

            # Update the number of consecutive failures, aborting the testsuite
            # if appropriate
            if item.result.status in (TestStatus.ERROR, TestStatus.FAIL):
                self.consecutive_failures += 1
                if (max_consecutive_failures > 0 and
                        self.consecutive_failures >= max_consecutive_failures):
                    self.aborted_too_many_failures = True
                    logger.error(
                        "Too many consecutive failures, aborting the testsuite"
                    )
                    raise KeyboardInterrupt
            else:
                self.consecutive_failures = 0

    def add_result(self, item: ResultQueueItem) -> None:
        """Add a test result to the result index and log it.

        :param item: Result queue item for the result to add.
        """
        assert self.main.args

        status = item.result.status
        test_name = item.result.test_name

        # The test results that reach this point are special: there were
        # serialized/deserialized through YAML, so the Log layer disappeared.
        assert status is not None

        # Ensure that we don't have two results with the same test name

        def indented_tb(tb: List[str]) -> str:
            return "".join("  {}".format(line) for line in tb)

        assert test_name not in self.report_index.entries, (
            f"cannot push twice results for {test_name}"
            f"\nFirst push happened at:"
            f"\n{indented_tb(self.result_tracebacks[test_name])}"
            f"\nThis one happened at:"
            f"\n{indented_tb(item.traceback)}")

        # Now that the result is validated, add it to our internals
        self.report_index.add_result(item.result, item.filename)
        self.result_tracebacks[test_name] = item.traceback
        self.running_status.set_status_counters(
            self.report_index.status_counters)
        if item.gaia_results:
            self.gaia_result_files[test_name] = item.gaia_results

        # Log the test result. If error output is requested and the test
        # failed unexpectedly, show the detailed logs.
        log_line = summary_line(item.result, self.colors,
                                self.main.args.show_time_info)
        if self.main.args.show_error_output and status not in (
                TestStatus.PASS,
                TestStatus.XFAIL,
                TestStatus.XPASS,
                TestStatus.SKIP,
        ):
            full_result = self.report_index.entries[test_name].load()

            def format_log(log: Log) -> str:
                return "\n" + str(log) + self.Style.RESET_ALL

            if full_result.diff:
                log_line += format_log(full_result.diff)
            else:
                log_line += format_log(full_result.log)
        logger.info(log_line)

    def add_test_error(self,
                       test_name: str,
                       message: str,
                       tb: Optional[str] = None) -> None:
        """Create and add an ERROR test status.

        :param test_name: Prefix for the test result to create. This adds a
            suffix to avoid clashes.
        :param str message: Error message.
        :param tb: Optional traceback for the error.
        """
        from e3.testsuite.driver import ResultQueueItem

        result = TestResult(
            f"{test_name}__except{len(self.report_index.entries)}",
            env={},
            status=TestStatus.ERROR,
            msg=message,
        )
        if tb:
            result.log += tb

        self.add_result(
            ResultQueueItem(
                result.summary,
                result.save(self.output_dir),
                traceback.format_stack(),
                dump_result_logs_if_needed(self.env, result, self.output_dir),
            ))

    def setup_result_dirs(self) -> None:
        """Create the output directory in which the results are stored."""
        assert self.main.args
        args = self.main.args

        # Both the actual new/old directories to use depend on both
        # --output-dir and --old-output-dir options.
        d = os.path.abspath(args.output_dir)
        if args.old_output_dir:
            self.output_dir = d
            old_output_dir = os.path.abspath(args.old_output_dir)
        else:
            self.output_dir = os.path.join(d, "new")
            old_output_dir = os.path.join(d, "old")

        # Rotate results directories if requested. In both cases, make sure the
        # new results dir is clean.
        if args.rotate_output_dirs:
            if os.path.isdir(old_output_dir):
                rm(old_output_dir, recursive=True)
            if os.path.isdir(self.output_dir):
                mv(self.output_dir, old_output_dir)
        elif os.path.isdir(self.output_dir):
            rm(self.output_dir, recursive=True)
        mkdir(self.output_dir)

        # Remember about the old output directory only if it exists and does
        # contain results. If not, this info will be unused at best, or lead to
        # incorrect behavior.
        self.old_output_dir = None
        if (os.path.exists(old_output_dir) and os.path.exists(
                os.path.join(old_output_dir, ReportIndex.INDEX_FILENAME))):
            self.old_output_dir = old_output_dir

        if args.dump_environ:
            with open(os.path.join(self.output_dir, "environ.sh"), "w") as f:
                for var_name in sorted(os.environ):
                    f.write("export {}={}\n".format(
                        var_name, quote_arg(os.environ[var_name])))

    def run_standard_mainloop(self, dag: DAG) -> None:
        """Run the main loop to execute test fragments in threads."""
        assert self.main.args is not None

        from e3.job import Job
        from e3.testsuite.fragment import FragmentData, ThreadTestFragment

        def job_factory(
            uid: str,
            data: Any,
            predecessors: FrozenSet[str],
            notify_end: Callable[[str], None],
        ) -> ThreadTestFragment:
            """Turn a DAG item into a ThreadTestFragment instance."""
            assert isinstance(data, FragmentData)

            # When passing return values from predecessors, remove current test
            # name from the keys to ease referencing by user (the short
            # fragment name can then be used by user without knowing the full
            # node id).
            key_prefix = data.driver.test_name + "."
            key_prefix_len = len(key_prefix)

            def filter_key(k: str) -> str:
                if k.startswith(key_prefix):
                    return k[key_prefix_len:]
                else:
                    return k

            return ThreadTestFragment(
                uid,
                data.driver,
                data.callback,
                {filter_key(k): self.return_values[k]
                 for k in predecessors},
                notify_end,
                self.running_status,
            )

        def collect_result(job: Job) -> bool:
            """Collect test results from the given fragment."""
            assert isinstance(job, ThreadTestFragment)
            self.return_values[job.uid] = job.return_value
            self.collect_result(job)

            # In the e3.job.scheduler API, collect returning "True" means
            # "requeue the job". We never want to do that.
            return False

        # Create a scheduler to run all fragments for the testsuite main loop
        scheduler = Scheduler(
            job_provider=job_factory,
            tokens=self.main.args.jobs,
            collect=collect_result,
        )

        # Run the tests. Note that when the testsuite aborts because of too
        # many consecutive test failures, we still want to produce a report and
        # exit through regular ways, to catch KeyboardInterrupt exceptions,
        # which e3's scheduler uses to abort the execution loop, but only in
        # such cases. In other words, let the exception propagates if it's the
        # user that interrupted the testsuite.
        try:
            scheduler.run(dag)
        except KeyboardInterrupt:
            if not self.aborted_too_many_failures:  # interactive-only
                raise

    def run_multiprocess_mainloop(self, dag: DAG) -> None:
        """Run the main loop to execute test fragments in subprocesses."""
        assert self.main.args is not None

        from e3.testsuite.fragment import FragmentData, ProcessTestFragment
        from e3.testsuite.multiprocess_scheduler import MultiprocessScheduler

        def job_factory(uid: str, data: FragmentData,
                        slot: int) -> ProcessTestFragment:
            """Turn a DAG item into a ProcessTestFragment instance."""
            assert data.callback_by_name
            return ProcessTestFragment(
                uid,
                data.driver,
                data.name,
                slot,
                self.running_status,
                self.env,
            )

        def collect_result(job: ProcessTestFragment) -> None:
            """Collect test results from the given fragment."""
            job.collect_result()
            self.collect_result(job)

        scheduler: MultiprocessScheduler[FragmentData, ProcessTestFragment] = (
            MultiprocessScheduler(dag,
                                  job_factory,
                                  collect_result,
                                  jobs=self.main.args.jobs))

        # See corresponding code/comment in run_multithread_mainloop
        try:
            scheduler.run()
        except KeyboardInterrupt:
            if not self.aborted_too_many_failures:  # interactive-only
                raise

    # Unlike the previous methods, the following ones are supposed to be
    # overriden.

    @property
    def tests_subdir(self) -> str:
        """
        Return the subdirectory in which tests are looked for.

        The returned directory name is considered relative to the root
        testsuite directory (self.root_dir).
        """
        raise NotImplementedError

    @property
    def test_driver_map(self) -> Dict[str, Type[TestDriver]]:
        """Return a map from test driver names to TestDriver subclasses.

        Test finders will be able to use this map to fetch the test drivers
        referenced in testcases.
        """
        raise NotImplementedError

    @property
    def default_driver(self) -> Optional[str]:
        """Return the name of the default driver for testcases.

        When tests do not query a specific driver, the one associated to this
        name is used instead. If this property returns None, all tests are
        required to query a driver.
        """
        raise NotImplementedError

    def test_name(self, test_dir: str) -> str:
        """Compute the test name given a testcase spec.

        This function can be overridden. By default it uses the name of the
        test directory. Note that the test name should be a valid filename (not
        dir seprators, or special characters such as ``:``, ...).
        """
        raise NotImplementedError

    @property
    def test_finders(self) -> List[TestFinder]:
        """Return test finders to probe tests directories."""
        raise NotImplementedError

    def add_options(self, parser: argparse.ArgumentParser) -> None:
        """Add testsuite specific switches.

        Subclasses can override this method to add their own testsuite
        command-line options.

        :param parser: Parser for command-line arguments. See
            <https://docs.python.org/3/library/argparse.html> for usage.
        """
        raise NotImplementedError

    def set_up(self) -> None:
        """Execute operations before running the testsuite.

        Before running this, command-line arguments were parsed. After this
        returns, the testsuite will look for testcases.

        By default, this does nothing. Overriding this method allows testsuites
        to prepare the execution of the testsuite depending on their needs. For
        instance:

        * process testsuite-specific options;
        * initialize environment variables;
        * adjust self.env (object forwarded to test drivers).
        """
        raise NotImplementedError

    def tear_down(self) -> None:
        """Execute operation when finalizing the testsuite.

        By default, this cleans the working (temporary) directory in which the
        tests were run.
        """
        raise NotImplementedError

    def write_comment_file(self, comment_file: IO[str]) -> None:
        """Write the comment file's content.

        :param comment_file: File descriptor for the comment file.  Overriding
            methods should only call its "write" method (or print to it).
        """
        raise NotImplementedError

    @property
    def default_max_consecutive_failures(self) -> int:
        """Return the default maximum number of consecutive failures.

        In some cases, aborting the testsuite when there are just too many
        failures saves time and costs: the software to test/environment is too
        broken, there is no point to continue running the testsuite.

        This property must return the number of test failures (FAIL or ERROR)
        that trigger the abortion of the testuite. If zero, this behavior is
        disabled.
        """
        raise NotImplementedError

    @property
    def default_failure_exit_code(self) -> int:
        """Return the default exit code when at least one test fails."""
        raise NotImplementedError

    @property
    def auto_generate_text_report(self) -> bool:
        """Return whether to automatically generate a text report.

        This is disabled by default (and controlled by the
        --generate-text-report command-line option) because the generation of
        this report can add non-trivial overhead depending on results.
        """
        raise NotImplementedError

    def adjust_dag_dependencies(self, dag: DAG) -> None:
        """Adjust dependencies in the DAG of all test fragments.

        :param dag: DAG to adjust.
        :param fragments: Set of all fragments added so far to the DAG.
        """
        raise NotImplementedError

    @property
    def multiprocessing_supported(self) -> bool:
        """Return whether running test fragments in subprocesses is supported.

        When multiprocessing is enabled (see the "use_multiprocessing"
        attribute), test fragments are executed in a separate process, and the
        propagation of their return values is disabled (FragmentData's
        "previous_values" argument is always an empty dict).

        This means that multiprocessing can work only if test drivers and all
        code used by test fragments can be imported by subprocesses (for
        instance, class defined in the testsuite entry point are unavailable)
        and if test drivers don't use the "previous_values" mechanism.

        Testsuite authors can use the "--force-multiprocessing" testsuite
        option to check if this works.
        """
        raise NotImplementedError
コード例 #12
0
        qualifier=m.args.qualifier,
        sandbox=sbx,
        upload=False,
        env=BaseEnv.from_env(),
    ).anod_instance

    saved_env = {k: v for k, v in os.environ.items()}

    if m.args.build_env:
        if hasattr(anod_instance, "build_setenv"):
            anod_instance.build_setenv()
    else:
        if hasattr(anod_instance, "setenv"):
            anod_instance.setenv()

    for var, value in os.environ.items():
        if var not in saved_env or saved_env[var] != os.environ[var]:
            print('export %s="%s";' % (var, value))

            if m.args.verbose >= 1:
                print('printf "I set %s=\\"%s\\"\\n\\n";' % (var, value))

            print(" ")

    print(BANNER % m.args.spec_name)
    return 0


if __name__ == "__main__":
    exit(do_printenv(Main(), set_prog=False))
コード例 #13
0
ファイル: __init__.py プロジェクト: Tubbz-alt/e3-testsuite
class TestsuiteCore:
    """Testsuite Core driver.

    This class is the base of Testsuite class and should not be instanciated.
    It's not recommended to override any of the functions declared in it.

    See documentation of Testsuite class for overridable methods and
    variables.
    """
    def __init__(self,
                 root_dir: Optional[str] = None,
                 testsuite_name: str = "Untitled testsute") -> None:
        """Testsuite constructor.

        :param root_dir: Root directory for the testsuite. If left to None, use
            the directory containing the Python module that created self's
            class.
        :param testsuite_name: Name for this testsuite. It can be used to
            provide a title in some report formats.
        """
        if root_dir is None:
            root_dir = os.path.dirname(inspect.getfile(type(self)))
        self.root_dir = os.path.abspath(root_dir)
        self.test_dir = os.path.join(self.root_dir, self.tests_subdir)
        logger.debug("Test directory: %s", self.test_dir)
        self.consecutive_failures = 0
        self.return_values: Dict[str, Any] = {}
        self.result_tracebacks: Dict[str, List[str]] = {}
        self.testsuite_name = testsuite_name

        self.aborted_too_many_failures = False
        """
        Whether the testsuite aborted because of too many consecutive test
        failures (see the --max-consecutive-failures command-line option).
        """

    # Mypy does not support decorators on properties, so keep the actual
    # implementations for deprecated properties in methods.

    @deprecated(2)
    def _test_counter(self) -> int:
        return len(self.report_index.entries)

    @deprecated(2)
    def _test_status_counters(self) -> Dict[TestStatus, int]:
        return self.report_index.status_counters

    @deprecated(2)
    def _results(self) -> Dict[str, TestStatus]:
        return {
            e.test_name: e.status
            for e in self.report_index.entries.values()
        }

    @property
    def test_counter(self) -> int:
        """Return the number of test results in the report.

        Warning: this method is obsolete and will be removed in the future.
        """
        return self._test_counter()

    @property
    def test_status_counters(self) -> Dict[TestStatus, int]:
        """Return test result counts per test status.

        Warning: this method is obsolete and will be removed in the future.
        """
        return self._test_status_counters()

    @property
    def results(self) -> Dict[str, TestStatus]:
        """Return a mapping from test names to results.

        Warning: this method is obsolete and will be removed in the future.
        """
        return self._results()

    def test_result_filename(self, test_name: str) -> str:
        """Return the name of the file in which the result are stored.

        :param test_name: Name of the test for this result file.
        """
        return os.path.join(self.output_dir, test_name + ".yaml")

    def job_factory(self, uid: str, data: Any, predecessors: FrozenSet[str],
                    notify_end: Callable[[str], None]) -> TestFragment:
        """Run internal function.

        See e3.job.scheduler
        """
        # We assume that data[0] is the test instance and data[1] the method
        # to call.

        # When passing return values from predecessors, remove current test
        # name from the keys to ease referencing by user (the short fragment
        # name can then be used by user without knowing the full node id).
        key_prefix = data[0].test_name + "."
        key_prefix_len = len(key_prefix)

        def filter_key(k: str) -> str:
            if k.startswith(key_prefix):
                return k[key_prefix_len:]
            else:
                return k

        return TestFragment(
            uid,
            data[0],
            data[1],
            {filter_key(k): self.return_values[k]
             for k in predecessors},
            notify_end,
        )

    def testsuite_main(self, args: Optional[List[str]] = None) -> int:
        """Main for the main testsuite script.

        :param args: Command line arguments. If None, use `sys.argv`.
        :return: The testsuite status code (0 for success, a positive for
            failure).
        """
        self.main = Main(platform_args=True)

        # Add common options
        parser = self.main.argument_parser
        parser.add_argument(
            "-o",
            "--output-dir",
            metavar="DIR",
            default="./out",
            help="select output dir",
        )
        parser.add_argument("-t",
                            "--temp-dir",
                            metavar="DIR",
                            default=Env().tmp_dir)
        parser.add_argument(
            "-d",
            "--dev-temp",
            nargs="?",
            default=None,
            const="tmp",
            help="Unlike --temp-dir, use this very directory to store"
            " testsuite temporaries (i.e. no random subdirectory). Also"
            " automatically disable temp dir cleanup, to be developer"
            " friendly. If no directory is provided, use the local"
            " \"tmp\" directory")
        parser.add_argument(
            "--max-consecutive-failures",
            "-M",
            metavar="N",
            type=int,
            default=self.default_max_consecutive_failures,
            help="Number of test failures (FAIL or ERROR) that trigger the"
            " abortion of the testuite. If zero, this behavior is disabled. In"
            " some cases, aborting the testsuite when there are just too many"
            " failures saves time and costs: the software to test/environment"
            " is too broken, there is no point to continue running the"
            " testsuite.")
        parser.add_argument(
            "--keep-old-output-dir",
            default=False,
            action="store_true",
            help="This is default with this testsuite framework. The option"
            " is kept only to keep backward compatibility of invocation with"
            " former framework (gnatpython.testdriver)",
        )
        parser.add_argument(
            "--disable-cleanup",
            dest="enable_cleanup",
            action="store_false",
            default=True,
            help="disable cleanup of working space",
        )
        parser.add_argument(
            "-j",
            "--jobs",
            dest="jobs",
            type=int,
            metavar="N",
            default=Env().build.cpu.cores,
            help="Specify the number of jobs to run simultaneously",
        )
        parser.add_argument(
            "--show-error-output",
            "-E",
            action="store_true",
            help="When testcases fail, display their output. This is for"
            " convenience for interactive use.",
        )
        parser.add_argument(
            "--show-time-info",
            action="store_true",
            help="Display time information for test results, if available")
        parser.add_argument(
            "--dump-environ",
            dest="dump_environ",
            action="store_true",
            default=False,
            help="Dump all environment variables in a file named environ.sh,"
            " located in the output directory (see --output-dir). This"
            " file can then be sourced from a Bourne shell to recreate"
            " the environement that existed when this testsuite was run"
            " to produce a given testsuite report.",
        )
        parser.add_argument(
            "--xunit-output",
            dest="xunit_output",
            metavar="FILE",
            help="Output testsuite report to the given file in the standard"
            " XUnit XML format. This is useful to display results in"
            " continuous build systems such as Jenkins.",
        )
        parser.add_argument(
            "--gaia-output",
            action="store_true",
            help="Output a GAIA-compatible testsuite report next to the YAML"
            " report.")
        parser.add_argument(
            "--truncate-logs",
            "-T",
            metavar="N",
            type=int,
            default=200,
            help="When outputs (for instance subprocess outputs) exceed 2*N"
            " lines, only include the first and last N lines in logs. This is"
            " necessary when storage for testsuite results have size limits,"
            " and the useful information is generally either at the beginning"
            " or the end of such outputs. If 0, never truncate logs.")
        parser.add_argument(
            "--failure-exit-code",
            metavar="N",
            type=int,
            default=self.default_failure_exit_code,
            help="Exit code the testsuite must use when at least one test"
            " result shows a failure/error. By default, this is"
            f" {self.default_failure_exit_code}. This option is useful when"
            " running a testsuite in a continuous integration setup, as this"
            " can make the testing process stop when there is a regression.")
        parser.add_argument("sublist",
                            metavar="tests",
                            nargs="*",
                            default=[],
                            help="test")
        # Add user defined options
        self.add_options(parser)

        # Parse options
        self.main.parse_args(args)
        assert self.main.args is not None

        # If there is a chance for the logging to end up in a non-tty stream,
        # disable colors. If not, be user-friendly and automatically show error
        # outputs.
        if (self.main.args.log_file or not isatty(sys.stdout)
                or not isatty(sys.stderr)):
            enable_colors = False
        else:  # interactive-only
            enable_colors = True
            self.main.args.show_error_output = True
        self.colors = ColorConfig(enable_colors)
        self.Fore = self.colors.Fore
        self.Style = self.colors.Style

        self.env = BaseEnv.from_env()
        self.env.enable_colors = enable_colors
        self.env.root_dir = self.root_dir
        self.env.test_dir = self.test_dir

        # At this stage compute commonly used paths Keep the working dir as
        # short as possible, to avoid the risk of having a path that's too long
        # (a problem often seen on Windows, or when using WRS tools that have
        # their own max path limitations).
        #
        # Note that we do make sure that working_dir is an absolute path, as we
        # are likely to be changing directories when running each test. A
        # relative path would no longer work under those circumstances.
        d = os.path.abspath(self.main.args.output_dir)
        self.output_dir = os.path.join(d, "new")
        self.old_output_dir = os.path.join(d, "old")

        if self.main.args.dev_temp:
            # Use a temporary directory for developers: make sure it is an
            # empty directory and disable cleanup to ease post-mortem
            # investigation.
            self.working_dir = os.path.abspath(self.main.args.dev_temp)
            rm(self.working_dir, recursive=True)
            mkdir(self.working_dir)
            self.main.args.enable_cleanup = False

        else:
            # If the temp dir is supposed to be randomized, we need to create a
            # subdirectory, so check that the parent directory exists first.
            if not os.path.isdir(self.main.args.temp_dir):
                logger.critical("temp dir '%s' does not exist",
                                self.main.args.temp_dir)
                return 1

            self.working_dir = tempfile.mkdtemp(
                "", "tmp", os.path.abspath(self.main.args.temp_dir))

        # Create the new output directory that will hold the results and create
        # an index for it.
        self.setup_result_dir()
        self.report_index = ReportIndex(self.output_dir)

        # Store in global env: target information and common paths
        self.env.output_dir = self.output_dir
        self.env.working_dir = self.working_dir
        self.env.options = self.main.args

        # User specific startup
        self.set_up()

        # Retrieve the list of test
        self.has_error = False
        self.test_list = self.get_test_list(self.main.args.sublist)

        # Launch the mainloop
        self.total_test = len(self.test_list)
        self.run_test = 0

        self.scheduler = Scheduler(
            job_provider=self.job_factory,
            tokens=self.main.args.jobs,

            # correct_result expects specifically TestFragment instances (a Job
            # subclass), while Scheduler only guarantees Job instances.
            # Test drivers are supposed to register only TestFragment
            # instances, so the following cast should be fine.
            collect=cast(Any, self.collect_result),
        )
        actions = DAG()
        for parsed_test in self.test_list:
            if not self.add_test(actions, parsed_test):
                self.has_error = True
        actions.check()

        with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
            fd.write(actions.as_dot())

        # Run the tests. Note that when the testsuite aborts because of too
        # many consecutive test failures, we still want to produce a report and
        # exit through regular ways, to catch KeyboardInterrupt exceptions,
        # which e3's scheduler uses to abort the execution loop, but only in
        # such cases. In other words, let the exception propagates if it's the
        # user that interrupted the testsuite.
        try:
            self.scheduler.run(actions)
        except KeyboardInterrupt:
            if not self.aborted_too_many_failures:  # interactive-only
                raise

        self.report_index.write()
        self.dump_testsuite_result()
        if self.main.args.xunit_output:
            dump_xunit_report(self, self.main.args.xunit_output)
        if self.main.args.gaia_output:
            dump_gaia_report(self, self.output_dir)

        # Clean everything
        self.tear_down()

        # Return the appropriate status code: 1 when there is a framework
        # issue, the failure status code from the --failure-exit-code=N option
        # when there is a least one testcase failure, or 0.
        statuses = {
            s
            for s, count in self.report_index.status_counters.items() if count
        }
        if self.has_error:
            return 1
        elif TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
            return self.main.args.failure_exit_code
        else:
            return 0

    def get_test_list(self, sublist: List[str]) -> List[ParsedTest]:
        """Retrieve the list of tests.

        :param sublist: A list of tests scenarios or patterns.
        """
        # Use a mapping: test name -> ParsedTest when building the result, as
        # several patterns in "sublist" may yield the same testcase.
        testcases: Dict[str, ParsedTest] = {}
        test_finders = self.test_finders

        def add_testcase(test: ParsedTest) -> None:
            testcases[test.test_name] = test

        def helper(spec: str) -> None:
            pattern: Optional[Pattern[str]] = None

            # If the given pattern is a directory, do not go through the whole
            # tests subdirectory.
            if os.path.isdir(spec):
                root = spec
            else:
                root = self.test_dir
                try:
                    pattern = re.compile(spec)
                except re.error as exc:
                    logger.debug(
                        "Test pattern is not a valid regexp, try to match it"
                        " as-is: {}".format(exc))
                    pattern = re.compile(re.escape(spec))

            # For each directory in the requested subdir, ask our test finders
            # to probe for a testcase. Register matches.
            for dirpath, dirnames, filenames in os.walk(root,
                                                        followlinks=True):
                # If the directory name does not match the given pattern, skip
                # it.
                if pattern is not None and not pattern.search(dirpath):
                    continue

                # The first test finder that has a match "wins". When handling
                # test data, we want to deal only with absolute paths, so get
                # the absolute name now.
                dirpath = os.path.abspath(dirpath)
                for tf in test_finders:
                    try:
                        test_or_list = tf.probe(self, dirpath, dirnames,
                                                filenames)
                    except ProbingError as exc:
                        self.has_error = True
                        logger.error(str(exc))
                        break
                    if isinstance(test_or_list, list):
                        for t in test_or_list:
                            add_testcase(t)
                        break
                    elif test_or_list is not None:
                        add_testcase(test_or_list)
                        break

        # If specific tests are requested, only look for them. Otherwise, just
        # look in the tests subdirectory.
        if sublist:
            for s in sublist:
                helper(s)
        else:
            helper(self.test_dir)

        result = list(testcases.values())
        logger.info("Found {} tests".format(len(result)))
        logger.debug("tests:\n  " + "\n  ".join(t.test_dir for t in result))
        return result

    def add_test(self, actions: DAG, parsed_test: ParsedTest) -> bool:
        """Register a test to run.

        :param actions: The dag of actions for the testsuite.
        :param parsed_test: Test to instantiate.
        :return: Whether the test was successfully registered.
        """
        test_name = parsed_test.test_name

        # Complete the test environment
        test_env = dict(parsed_test.test_env)
        test_env["test_dir"] = parsed_test.test_dir
        test_env["test_name"] = test_name

        assert isinstance(self.env.working_dir, str)
        test_env["working_dir"] = os.path.join(self.env.working_dir, test_name)

        # Fetch the test driver to use
        driver = parsed_test.driver_cls
        if not driver:
            if self.default_driver:
                driver = self.test_driver_map[self.default_driver]
            else:
                logger.error("missing driver for test '{}'".format(test_name))
                return False

        # Finally run the driver instantiation
        try:
            instance = driver(self.env, test_env)
            instance.Fore = self.Fore
            instance.Style = self.Style
            instance.add_test(actions)

        except Exception as e:
            error_msg = str(e)
            error_msg += "\nTraceback:\n"
            error_msg += "\n".join(traceback.format_tb(sys.exc_info()[2]))
            logger.error(error_msg)
            return False

        return True

    def dump_testsuite_result(self) -> None:
        """Log a summary of test results.

        Subclasses are free to override this to do whatever is suitable for
        them.
        """
        lines = ['Summary:']

        # Display test count for each status, but only for status that have
        # at least one test. Sort them by status value, to get consistent
        # order.
        def sort_key(couple: Tuple[TestStatus, int]) -> Any:
            status, _ = couple
            return status.value

        stats = sorted(
            ((status, count)
             for status, count in self.report_index.status_counters.items()
             if count),
            key=sort_key)
        for status, count in stats:
            lines.append('  {}{: <12}{} {}'.format(status.color(self.colors),
                                                   status.name,
                                                   self.Style.RESET_ALL,
                                                   count))
        if not stats:
            lines.append('  <no test result>')
        logger.info('\n'.join(lines))

        # Dump the comment file
        with open(os.path.join(self.output_dir, "comment"), "w") as f:
            self.write_comment_file(f)

    def collect_result(self, job: TestFragment) -> bool:
        """Run internal function.

        :param job: A job that is finished.
        """
        assert self.main.args

        # Keep track of the number of consecutive failures seen so far if it
        # reaches the maximum number allowed, we must abort the testsuite.
        max_consecutive_failures = self.main.args.max_consecutive_failures
        consecutive_failures = 0

        self.return_values[job.uid] = job.return_value

        while job.test_instance.result_queue:
            result, tb = job.test_instance.result_queue.pop()

            # The test results that reach this point are special: there were
            # serialized/deserialized through YAML, so the Log layer
            # disappeared.
            assert result.status is not None

            # Log the test result. If error output is requested and the test
            # failed unexpectedly, show the detailed logs.
            log_line = summary_line(result, self.colors,
                                    self.main.args.show_time_info)
            if (self.main.args.show_error_output and result.status
                    not in (TestStatus.PASS, TestStatus.XFAIL,
                            TestStatus.XPASS, TestStatus.SKIP)):

                def format_log(log: Log) -> str:
                    return "\n" + str(log) + self.Style.RESET_ALL

                if result.diff:
                    log_line += format_log(result.diff)
                else:
                    log_line += format_log(result.log)
            logger.info(log_line)

            def indented_tb(tb: List[str]) -> str:
                return "".join("  {}".format(line) for line in tb)

            assert result.test_name not in self.report_index.entries, (
                "cannot push twice results for {}"
                "\nFirst push happened at:"
                "\n{}"
                "\nThis one happened at:"
                "\n{}".format(
                    result.test_name,
                    indented_tb(self.result_tracebacks[result.test_name]),
                    indented_tb(tb),
                ))
            with open(self.test_result_filename(result.test_name), "w") as fd:
                yaml.dump(result, fd)
            self.report_index.add_result(result)
            self.result_tracebacks[result.test_name] = tb

            # Update the number of consecutive failures, aborting the testsuite
            # if appropriate
            if result.status in (TestStatus.ERROR, TestStatus.FAIL):
                consecutive_failures += 1
                if (max_consecutive_failures > 0
                        and consecutive_failures >= max_consecutive_failures):
                    self.aborted_too_many_failures = True
                    logger.error(
                        "Too many consecutive failures, aborting the testsuite"
                    )
                    raise KeyboardInterrupt
            else:
                consecutive_failures = 0

        return False

    def setup_result_dir(self) -> None:
        """Create the output directory in which the results are stored."""
        assert self.main.args

        if os.path.isdir(self.old_output_dir):
            rm(self.old_output_dir, True)
        if os.path.isdir(self.output_dir):
            mv(self.output_dir, self.old_output_dir)
        mkdir(self.output_dir)

        if self.main.args.dump_environ:
            with open(os.path.join(self.output_dir, "environ.sh"), "w") as f:
                for var_name in sorted(os.environ):
                    f.write("export {}={}\n".format(
                        var_name, quote_arg(os.environ[var_name])))

    # Unlike the previous methods, the following ones are supposed to be
    # overriden.

    @property
    def tests_subdir(self) -> str:
        """
        Return the subdirectory in which tests are looked for.

        The returned directory name is considered relative to the root
        testsuite directory (self.root_dir).
        """
        raise NotImplementedError

    @property
    def test_driver_map(self) -> Dict[str, Type[TestDriver]]:
        """Return a map from test driver names to TestDriver subclasses.

        Test finders will be able to use this map to fetch the test drivers
        referenced in testcases.
        """
        raise NotImplementedError

    @property
    def default_driver(self) -> Optional[str]:
        """Return the name of the default driver for testcases.

        When tests do not query a specific driver, the one associated to this
        name is used instead. If this property returns None, all tests are
        required to query a driver.
        """
        raise NotImplementedError

    def test_name(self, test_dir: str) -> str:
        """Compute the test name given a testcase spec.

        This function can be overridden. By default it uses the name of the
        test directory. Note that the test name should be a valid filename (not
        dir seprators, or special characters such as ``:``, ...).
        """
        raise NotImplementedError

    @property
    def test_finders(self) -> List[TestFinder]:
        """Return test finders to probe tests directories."""
        raise NotImplementedError

    def add_options(self, parser: argparse.ArgumentParser) -> None:
        """Add testsuite specific switches.

        Subclasses can override this method to add their own testsuite
        command-line options.

        :param parser: Parser for command-line arguments. See
            <https://docs.python.org/3/library/argparse.html> for usage.
        """
        raise NotImplementedError

    def set_up(self) -> None:
        """Execute operations before running the testsuite.

        Before running this, command-line arguments were parsed. After this
        returns, the testsuite will look for testcases.

        By default, this does nothing. Overriding this method allows testsuites
        to prepare the execution of the testsuite depending on their needs. For
        instance:

        * process testsuite-specific options;
        * initialize environment variables;
        * adjust self.env (object forwarded to test drivers).
        """
        raise NotImplementedError

    def tear_down(self) -> None:
        """Execute operation when finalizing the testsuite.

        By default, this cleans the working (temporary) directory in which the
        tests were run.
        """
        raise NotImplementedError

    def write_comment_file(self, comment_file: IO[str]) -> None:
        """Write the comment file's content.

        :param comment_file: File descriptor for the comment file.  Overriding
            methods should only call its "write" method (or print to it).
        """
        raise NotImplementedError

    @property
    def default_max_consecutive_failures(self) -> int:
        """Return the default maximum number of consecutive failures.

        In some cases, aborting the testsuite when there are just too many
        failures saves time and costs: the software to test/environment is too
        broken, there is no point to continue running the testsuite.

        This property must return the number of test failures (FAIL or ERROR)
        that trigger the abortion of the testuite. If zero, this behavior is
        disabled.
        """
        raise NotImplementedError

    @property
    def default_failure_exit_code(self) -> int:
        """Return the default exit code when at least one test fails."""
        raise NotImplementedError
コード例 #14
0
class TestsuiteCore(object):
    """Testsuite Core driver.

    This class is the base of Testsuite class and should not be instanciated.
    It's not recommended to override any of the functions declared in it.

    See documentation of Testsuite class for overridable methods and
    variables.
    """
    def __init__(self, root_dir):
        """Testsuite constructor.

        :param root_dir: root dir of the testsuite. Usually the directory in
            which testsuite.py and runtest.py are located
        :type root_dir: str | unicode
        """
        self.root_dir = os.path.abspath(root_dir)
        self.test_dir = os.path.join(self.root_dir, self.TEST_SUBDIR)
        self.consecutive_failures = 0
        self.return_values = {}
        self.results = {}
        self.test_counter = 0
        self.test_status_counters = {s: 0 for s in TestStatus}

    def test_result_filename(self, test_name):
        """Return the name of the file in which the result are stored.

        :param test_case_file: path to a test case scenario relative to the
            test directory
        :type test_case_file: str | unicode
        :param variant: the test variant
        :type variant: str
        :return: the test name. Note that test names should not contain path
            separators
        :rtype: str | unicode
        """
        return os.path.join(self.output_dir, test_name + '.yaml')

    def job_factory(self, uid, data, predecessors, notify_end):
        """Run internal function.

        See e3.job.scheduler
        """
        # we assume that data[0] is the test instance and data[1] the method
        # to call

        # When passing return values from predecessors, remove current test
        # name from the keys to ease referencing by user (the short fragment
        # name can then be used by user without knowing the full node id).
        key_prefix = data[0].test_name + '.'
        key_prefix_len = len(key_prefix)

        def filter_key(k):
            if k.startswith(key_prefix):
                return k[key_prefix_len:]
            else:
                return k

        return TestFragment(
            uid, data[0], data[1],
            {filter_key(k): self.return_values[k]
             for k in predecessors}, notify_end)

    def testsuite_main(self, args=None):
        """Main for the main testsuite script.

        :param args: command line arguments. If None use sys.argv
        :type args: list[str] | None
        """
        self.main = Main(platform_args=self.CROSS_SUPPORT)

        # Add common options
        parser = self.main.argument_parser
        parser.add_argument("-o",
                            "--output-dir",
                            metavar="DIR",
                            default="./out",
                            help="select output dir")
        parser.add_argument("-t",
                            "--temp-dir",
                            metavar="DIR",
                            default=Env().tmp_dir)
        parser.add_argument(
            "--max-consecutive-failures",
            default=0,
            help="If there are more than N consecutive failures, the testsuite"
            " is aborted. If set to 0 (default) then the testsuite will never"
            " be stopped")
        parser.add_argument(
            "--keep-old-output-dir",
            default=False,
            action="store_true",
            help="This is default with this testsuite framework. The option"
            " is kept only to keep backward compatibility of invocation with"
            " former framework (gnatpython.testdriver)")
        parser.add_argument("--disable-cleanup",
                            dest="enable_cleanup",
                            action="store_false",
                            default=True,
                            help="disable cleanup of working space")
        parser.add_argument(
            "-j",
            "--jobs",
            dest="jobs",
            type=int,
            metavar="N",
            default=Env().build.cpu.cores,
            help="Specify the number of jobs to run simultaneously")
        parser.add_argument(
            "--show-error-output",
            action="store_true",
            help="When testcases fail, display their output. This is for"
            " convenience for interactive use.")
        parser.add_argument(
            "--dump-environ",
            dest="dump_environ",
            action="store_true",
            default=False,
            help="Dump all environment variables in a file named environ.sh,"
            " located in the output directory (see --output-dir). This"
            " file can then be sourced from a Bourne shell to recreate"
            " the environement that existed when this testsuite was run"
            " to produce a given testsuite report.")
        parser.add_argument('sublist',
                            metavar='tests',
                            nargs='*',
                            default=[],
                            help='test')
        # Add user defined options
        self.add_options()

        # parse options
        self.main.parse_args(args)

        self.env = BaseEnv.from_env()
        self.env.root_dir = self.root_dir
        self.env.test_dir = self.test_dir

        # At this stage compute commonly used paths
        # Keep the working dir as short as possible, to avoid the risk
        # of having a path that's too long (a problem often seen on
        # Windows, or when using WRS tools that have their own max path
        # limitations).
        # Note that we do make sure that working_dir is an absolute
        # path, as we are likely to be changing directories when
        # running each test. A relative path would no longer work
        # under those circumstances.
        d = os.path.abspath(self.main.args.output_dir)
        self.output_dir = os.path.join(d, 'new')
        self.old_output_dir = os.path.join(d, 'old')

        if not os.path.isdir(self.main.args.temp_dir):
            logging.critical("temp dir '%s' does not exist",
                             self.main.args.temp_dir)
            return 1

        self.working_dir = tempfile.mkdtemp(
            '', 'tmp', os.path.abspath(self.main.args.temp_dir))

        # Create the new output directory that will hold the results
        self.setup_result_dir()

        # Store in global env: target information and common paths
        self.env.output_dir = self.output_dir
        self.env.working_dir = self.working_dir
        self.env.options = self.main.args

        # User specific startup
        self.tear_up()

        # Retrieve the list of test
        self.test_list = self.get_test_list(self.main.args.sublist)

        # Launch the mainloop
        self.total_test = len(self.test_list)
        self.run_test = 0

        self.scheduler = Scheduler(job_provider=self.job_factory,
                                   collect=self.collect_result,
                                   tokens=self.main.args.jobs)
        actions = DAG()
        for test in self.test_list:
            self.parse_test(actions, test)

        with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd:
            fd.write(actions.as_dot())
        self.scheduler.run(actions)

        self.dump_testsuite_result()

        # Clean everything
        self.tear_down()
        return 0

    def parse_test(self, actions, test_case_file):
        """Register a test.

        :param actions: the dag of actions for the testsuite
        :type actions: e3.collection.dag.DAG
        :param test_case_file: filename containing the testcase
        :type test_case_file: str
        """
        # Load testcase file
        test_env = load_with_config(
            os.path.join(self.test_dir, test_case_file),
            Env().to_dict())

        # Ensure that the test_env act like a dictionary
        if not isinstance(test_env, collections.Mapping):
            test_env = {
                'test_name': self.test_name(test_case_file),
                'test_yaml_wrong_content': test_env
            }
            logger.error("abort test because of invalid test.yaml")
            return

        # Add to the test environment the directory in which the test.yaml is
        # stored
        test_env['test_dir'] = os.path.join(self.env.test_dir,
                                            os.path.dirname(test_case_file))
        test_env['test_case_file'] = test_case_file
        test_env['test_name'] = self.test_name(test_case_file)
        test_env['working_dir'] = os.path.join(self.env.working_dir,
                                               test_env['test_name'])

        if 'driver' in test_env:
            driver = test_env['driver']
        else:
            driver = self.default_driver

        logger.debug('set driver to %s' % driver)
        if driver not in self.DRIVERS or \
                not issubclass(self.DRIVERS[driver], TestDriver):
            logger.error('cannot find driver for %s' % test_case_file)
            return

        try:
            instance = self.DRIVERS[driver](self.env, test_env)
            instance.add_test(actions)

        except Exception as e:
            error_msg = str(e)
            error_msg += "Traceback:\n"
            error_msg += "\n".join(traceback.format_tb(sys.exc_traceback))
            logger.error(error_msg)
            return

    def dump_testsuite_result(self):
        """To be implemented."""
        pass

    def collect_result(self, job):
        """Run internal function.

        :param job: a job that is finished
        :type job: TestFragment
        """
        self.return_values[job.uid] = job.return_value
        while job.test_instance.result_queue:
            result = job.test_instance.result_queue.pop()
            logging.info('%-12s %s' % (str(result.status), result.test_name))
            assert result.test_name not in self.results, \
                'cannot push twice results for %s' % result.test_name
            with open(self.test_result_filename(result.test_name), 'wb') as fd:
                yaml.dump(result, fd)
            self.results[result.test_name] = result.status
            self.test_counter += 1
            self.test_status_counters[result.status] += 1
        return False

    def setup_result_dir(self):
        """Create the output directory in which the results are stored."""
        if os.path.isdir(self.old_output_dir):
            rm(self.old_output_dir, True)
        if os.path.isdir(self.output_dir):
            mv(self.output_dir, self.old_output_dir)
        mkdir(self.output_dir)

        if self.main.args.dump_environ:
            with open(os.path.join(self.output_dir, 'environ.sh'), 'w') as f:
                for var_name in sorted(os.environ):
                    f.write('export %s=%s\n' %
                            (var_name, quote_arg(os.environ[var_name])))
コード例 #15
0
    def testsuite_main(self, args=None):
        """Main for the main testsuite script.

        :param args: command line arguments. If None use sys.argv
        :type args: list[str] | None
        """
        self.main = Main(platform_args=self.CROSS_SUPPORT)

        # Add common options
        parser = self.main.argument_parser
        parser.add_argument("-o",
                            "--output-dir",
                            metavar="DIR",
                            default="./out",
                            help="select output dir")
        parser.add_argument("-t",
                            "--temp-dir",
                            metavar="DIR",
                            default=Env().tmp_dir)
        parser.add_argument(
            "--max-consecutive-failures",
            default=0,
            help="If there are more than N consecutive failures, the testsuite"
            " is aborted. If set to 0 (default) then the testsuite will never"
            " be stopped")
        parser.add_argument(
            "--keep-old-output-dir",
            default=False,
            action="store_true",
            help="This is default with this testsuite framework. The option"
            " is kept only to keep backward compatibility of invocation with"
            " former framework (gnatpython.testdriver)")
        parser.add_argument("--disable-cleanup",
                            dest="enable_cleanup",
                            action="store_false",
                            default=True,
                            help="disable cleanup of working space")
        parser.add_argument(
            "-j",
            "--jobs",
            dest="jobs",
            type=int,
            metavar="N",
            default=Env().build.cpu.cores,
            help="Specify the number of jobs to run simultaneously")
        parser.add_argument(
            "--show-error-output",
            action="store_true",
            help="When testcases fail, display their output. This is for"
            " convenience for interactive use.")
        parser.add_argument(
            "--dump-environ",
            dest="dump_environ",
            action="store_true",
            default=False,
            help="Dump all environment variables in a file named environ.sh,"
            " located in the output directory (see --output-dir). This"
            " file can then be sourced from a Bourne shell to recreate"
            " the environement that existed when this testsuite was run"
            " to produce a given testsuite report.")
        parser.add_argument('sublist',
                            metavar='tests',
                            nargs='*',
                            default=[],
                            help='test')
        # Add user defined options
        self.add_options()

        # parse options
        self.main.parse_args(args)

        self.env = BaseEnv.from_env()
        self.env.root_dir = self.root_dir
        self.env.test_dir = self.test_dir

        # At this stage compute commonly used paths
        # Keep the working dir as short as possible, to avoid the risk
        # of having a path that's too long (a problem often seen on
        # Windows, or when using WRS tools that have their own max path
        # limitations).
        # Note that we do make sure that working_dir is an absolute
        # path, as we are likely to be changing directories when
        # running each test. A relative path would no longer work
        # under those circumstances.
        d = os.path.abspath(self.main.args.output_dir)
        self.output_dir = os.path.join(d, 'new')
        self.old_output_dir = os.path.join(d, 'old')

        if not os.path.isdir(self.main.args.temp_dir):
            logging.critical("temp dir '%s' does not exist",
                             self.main.args.temp_dir)
            return 1

        self.working_dir = tempfile.mkdtemp(
            '', 'tmp', os.path.abspath(self.main.args.temp_dir))

        # Create the new output directory that will hold the results
        self.setup_result_dir()

        # Store in global env: target information and common paths
        self.env.output_dir = self.output_dir
        self.env.working_dir = self.working_dir
        self.env.options = self.main.args

        # User specific startup
        self.tear_up()

        # Retrieve the list of test
        self.test_list = self.get_test_list(self.main.args.sublist)

        # Launch the mainloop
        self.total_test = len(self.test_list)
        self.run_test = 0

        self.scheduler = Scheduler(job_provider=self.job_factory,
                                   collect=self.collect_result,
                                   tokens=self.main.args.jobs)
        actions = DAG()
        for test in self.test_list:
            self.parse_test(actions, test)

        with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd:
            fd.write(actions.as_dot())
        self.scheduler.run(actions)

        self.dump_testsuite_result()

        # Clean everything
        self.tear_down()
        return 0
コード例 #16
0
ファイル: scripts.py プロジェクト: AdaCore/e3-core
def anod():
    """bin/anod script entry point.

    This script is installed in the sandbox.
    """
    import os
    import sys

    import e3.anod.driver
    import e3.anod.loader
    import e3.anod.sandbox
    import e3.env
    import e3.store
    import e3.store.cache

    sandbox_dir = os.path.abspath(os.path.join(os.path.dirname(
        sys.modules['__main__'].__file__), os.pardir))

    sandbox = e3.anod.sandbox.SandBox()
    sandbox.root_dir = sandbox_dir

    # Load the local specs
    spec_repo = e3.anod.loader.AnodSpecRepository(
        os.path.join(sandbox_dir, 'specs'))

    # Load the cache
    cache = e3.store.cache.load_cache(
        'file-cache',
        {'cache_dir': sandbox.tmp_cache_dir})

    store = e3.store.load_store(
        'http-simple-store', {}, cache)

    m = Main()
    subparsers = m.argument_parser.add_subparsers()
    anod_cmdline(subparsers, 'download', 'download a binary package')
    m.parse_args()

    action = m.args.action_name
    spec = m.args.spec
    qualifier = m.args.qualifier

    anod_cls = spec_repo.load(name=spec)
    anod_instance = anod_cls(qualifier=qualifier,
                             kind=action,
                             jobs=1,
                             env=e3.env.BaseEnv.from_env())

    # ??? inject the sandbox
    anod_instance.sandbox = sandbox

    driver = e3.anod.driver.AnodDriver(
        anod_instance=anod_instance,
        store=store)

    try:
        driver.activate()
        driver.call(action)
    except AnodError as err:
        print(err, file=sys.stderr)
        sys.exit(1)
コード例 #17
0
ファイル: __init__.py プロジェクト: Tubbz-alt/e3-testsuite
    def testsuite_main(self, args: Optional[List[str]] = None) -> int:
        """Main for the main testsuite script.

        :param args: Command line arguments. If None, use `sys.argv`.
        :return: The testsuite status code (0 for success, a positive for
            failure).
        """
        self.main = Main(platform_args=True)

        # Add common options
        parser = self.main.argument_parser
        parser.add_argument(
            "-o",
            "--output-dir",
            metavar="DIR",
            default="./out",
            help="select output dir",
        )
        parser.add_argument("-t",
                            "--temp-dir",
                            metavar="DIR",
                            default=Env().tmp_dir)
        parser.add_argument(
            "-d",
            "--dev-temp",
            nargs="?",
            default=None,
            const="tmp",
            help="Unlike --temp-dir, use this very directory to store"
            " testsuite temporaries (i.e. no random subdirectory). Also"
            " automatically disable temp dir cleanup, to be developer"
            " friendly. If no directory is provided, use the local"
            " \"tmp\" directory")
        parser.add_argument(
            "--max-consecutive-failures",
            "-M",
            metavar="N",
            type=int,
            default=self.default_max_consecutive_failures,
            help="Number of test failures (FAIL or ERROR) that trigger the"
            " abortion of the testuite. If zero, this behavior is disabled. In"
            " some cases, aborting the testsuite when there are just too many"
            " failures saves time and costs: the software to test/environment"
            " is too broken, there is no point to continue running the"
            " testsuite.")
        parser.add_argument(
            "--keep-old-output-dir",
            default=False,
            action="store_true",
            help="This is default with this testsuite framework. The option"
            " is kept only to keep backward compatibility of invocation with"
            " former framework (gnatpython.testdriver)",
        )
        parser.add_argument(
            "--disable-cleanup",
            dest="enable_cleanup",
            action="store_false",
            default=True,
            help="disable cleanup of working space",
        )
        parser.add_argument(
            "-j",
            "--jobs",
            dest="jobs",
            type=int,
            metavar="N",
            default=Env().build.cpu.cores,
            help="Specify the number of jobs to run simultaneously",
        )
        parser.add_argument(
            "--show-error-output",
            "-E",
            action="store_true",
            help="When testcases fail, display their output. This is for"
            " convenience for interactive use.",
        )
        parser.add_argument(
            "--show-time-info",
            action="store_true",
            help="Display time information for test results, if available")
        parser.add_argument(
            "--dump-environ",
            dest="dump_environ",
            action="store_true",
            default=False,
            help="Dump all environment variables in a file named environ.sh,"
            " located in the output directory (see --output-dir). This"
            " file can then be sourced from a Bourne shell to recreate"
            " the environement that existed when this testsuite was run"
            " to produce a given testsuite report.",
        )
        parser.add_argument(
            "--xunit-output",
            dest="xunit_output",
            metavar="FILE",
            help="Output testsuite report to the given file in the standard"
            " XUnit XML format. This is useful to display results in"
            " continuous build systems such as Jenkins.",
        )
        parser.add_argument(
            "--gaia-output",
            action="store_true",
            help="Output a GAIA-compatible testsuite report next to the YAML"
            " report.")
        parser.add_argument(
            "--truncate-logs",
            "-T",
            metavar="N",
            type=int,
            default=200,
            help="When outputs (for instance subprocess outputs) exceed 2*N"
            " lines, only include the first and last N lines in logs. This is"
            " necessary when storage for testsuite results have size limits,"
            " and the useful information is generally either at the beginning"
            " or the end of such outputs. If 0, never truncate logs.")
        parser.add_argument(
            "--failure-exit-code",
            metavar="N",
            type=int,
            default=self.default_failure_exit_code,
            help="Exit code the testsuite must use when at least one test"
            " result shows a failure/error. By default, this is"
            f" {self.default_failure_exit_code}. This option is useful when"
            " running a testsuite in a continuous integration setup, as this"
            " can make the testing process stop when there is a regression.")
        parser.add_argument("sublist",
                            metavar="tests",
                            nargs="*",
                            default=[],
                            help="test")
        # Add user defined options
        self.add_options(parser)

        # Parse options
        self.main.parse_args(args)
        assert self.main.args is not None

        # If there is a chance for the logging to end up in a non-tty stream,
        # disable colors. If not, be user-friendly and automatically show error
        # outputs.
        if (self.main.args.log_file or not isatty(sys.stdout)
                or not isatty(sys.stderr)):
            enable_colors = False
        else:  # interactive-only
            enable_colors = True
            self.main.args.show_error_output = True
        self.colors = ColorConfig(enable_colors)
        self.Fore = self.colors.Fore
        self.Style = self.colors.Style

        self.env = BaseEnv.from_env()
        self.env.enable_colors = enable_colors
        self.env.root_dir = self.root_dir
        self.env.test_dir = self.test_dir

        # At this stage compute commonly used paths Keep the working dir as
        # short as possible, to avoid the risk of having a path that's too long
        # (a problem often seen on Windows, or when using WRS tools that have
        # their own max path limitations).
        #
        # Note that we do make sure that working_dir is an absolute path, as we
        # are likely to be changing directories when running each test. A
        # relative path would no longer work under those circumstances.
        d = os.path.abspath(self.main.args.output_dir)
        self.output_dir = os.path.join(d, "new")
        self.old_output_dir = os.path.join(d, "old")

        if self.main.args.dev_temp:
            # Use a temporary directory for developers: make sure it is an
            # empty directory and disable cleanup to ease post-mortem
            # investigation.
            self.working_dir = os.path.abspath(self.main.args.dev_temp)
            rm(self.working_dir, recursive=True)
            mkdir(self.working_dir)
            self.main.args.enable_cleanup = False

        else:
            # If the temp dir is supposed to be randomized, we need to create a
            # subdirectory, so check that the parent directory exists first.
            if not os.path.isdir(self.main.args.temp_dir):
                logger.critical("temp dir '%s' does not exist",
                                self.main.args.temp_dir)
                return 1

            self.working_dir = tempfile.mkdtemp(
                "", "tmp", os.path.abspath(self.main.args.temp_dir))

        # Create the new output directory that will hold the results and create
        # an index for it.
        self.setup_result_dir()
        self.report_index = ReportIndex(self.output_dir)

        # Store in global env: target information and common paths
        self.env.output_dir = self.output_dir
        self.env.working_dir = self.working_dir
        self.env.options = self.main.args

        # User specific startup
        self.set_up()

        # Retrieve the list of test
        self.has_error = False
        self.test_list = self.get_test_list(self.main.args.sublist)

        # Launch the mainloop
        self.total_test = len(self.test_list)
        self.run_test = 0

        self.scheduler = Scheduler(
            job_provider=self.job_factory,
            tokens=self.main.args.jobs,

            # correct_result expects specifically TestFragment instances (a Job
            # subclass), while Scheduler only guarantees Job instances.
            # Test drivers are supposed to register only TestFragment
            # instances, so the following cast should be fine.
            collect=cast(Any, self.collect_result),
        )
        actions = DAG()
        for parsed_test in self.test_list:
            if not self.add_test(actions, parsed_test):
                self.has_error = True
        actions.check()

        with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
            fd.write(actions.as_dot())

        # Run the tests. Note that when the testsuite aborts because of too
        # many consecutive test failures, we still want to produce a report and
        # exit through regular ways, to catch KeyboardInterrupt exceptions,
        # which e3's scheduler uses to abort the execution loop, but only in
        # such cases. In other words, let the exception propagates if it's the
        # user that interrupted the testsuite.
        try:
            self.scheduler.run(actions)
        except KeyboardInterrupt:
            if not self.aborted_too_many_failures:  # interactive-only
                raise

        self.report_index.write()
        self.dump_testsuite_result()
        if self.main.args.xunit_output:
            dump_xunit_report(self, self.main.args.xunit_output)
        if self.main.args.gaia_output:
            dump_gaia_report(self, self.output_dir)

        # Clean everything
        self.tear_down()

        # Return the appropriate status code: 1 when there is a framework
        # issue, the failure status code from the --failure-exit-code=N option
        # when there is a least one testcase failure, or 0.
        statuses = {
            s
            for s, count in self.report_index.status_counters.items() if count
        }
        if self.has_error:
            return 1
        elif TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
            return self.main.args.failure_exit_code
        else:
            return 0
コード例 #18
0
ファイル: install_cygwin.py プロジェクト: filin8804/gsh
    c.add_package(options.package)
    if not options.dry_run:
        c.install_all()


def remove_package_cmd(options):
    cm = CygwinMirror(options.download_dir)
    si = CygwinSetupIni(cm, options.download_dir, options.version)
    c = Cygwin(si)
    c.remove_package(options.package)
    if not options.dry_run:
        c.install_all()


if __name__ == '__main__':
    m = Main()
    parser = m.argument_parser
    parser.description = DESCRIPTION

    # Ensure that if we have already an installation then we used the same
    # version (32bits or 64bits)
    if os.path.isfile('c:/cygwin/etc/rebase.db.x86_64'):
        default_version = 'x86_64'
    elif os.path.isfile('c:/cygwin/etc/rebase.db.x86'):
        default_version = 'x86'
    else:
        default_version = 'x86_64'

    parser.add_argument(
        "--download-dir",
        default='c:\\cygwin_packages',