Пример #1
0
def get_loader(test_paths, product, ssl_env, debug=None, **kwargs):
    run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)

    test_manifests = testloader.ManifestLoader(
        test_paths, force_manifest_update=kwargs["manifest_update"]).load()

    manifest_filters = []
    meta_filters = []

    if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
        manifest_filters.append(
            testloader.TestFilter(include=kwargs["include"],
                                  exclude=kwargs["exclude"],
                                  manifest_path=kwargs["include_manifest"],
                                  test_manifests=test_manifests))
    if kwargs["tags"]:
        meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))

    test_loader = testloader.TestLoader(test_manifests,
                                        kwargs["test_types"],
                                        run_info,
                                        manifest_filters=manifest_filters,
                                        meta_filters=meta_filters,
                                        chunk_type=kwargs["chunk_type"],
                                        total_chunks=kwargs["total_chunks"],
                                        chunk_number=kwargs["this_chunk"],
                                        include_https=ssl_env.ssl_enabled)
    return run_info, test_loader
Пример #2
0
def get_loader(test_paths, product, ssl_env, debug=None, run_info_extras=None, **kwargs):
    if run_info_extras is None:
        run_info_extras = {}

    run_info = wpttest.get_run_info(kwargs["run_info"], product,
                                    browser_version=kwargs.get("browser_version"),
                                    debug=debug,
                                    extras=run_info_extras)

    test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"],
                                               manifest_download=kwargs["manifest_download"]).load()

    manifest_filters = []
    meta_filters = []

    if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
        manifest_filters.append(testloader.TestFilter(include=kwargs["include"],
                                                      exclude=kwargs["exclude"],
                                                      manifest_path=kwargs["include_manifest"],
                                                      test_manifests=test_manifests))
    if kwargs["tags"]:
        meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))

    test_loader = testloader.TestLoader(test_manifests,
                                        kwargs["test_types"],
                                        run_info,
                                        manifest_filters=manifest_filters,
                                        meta_filters=meta_filters,
                                        chunk_type=kwargs["chunk_type"],
                                        total_chunks=kwargs["total_chunks"],
                                        chunk_number=kwargs["this_chunk"],
                                        include_https=ssl_env.ssl_enabled,
                                        skip_timeout=kwargs["skip_timeout"])
    return run_info, test_loader
Пример #3
0
def list_disabled(serve_root, test_paths, test_types, product, **kwargs):
    do_delayed_imports(serve_root)
    rv = []
    run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
    test_loader = testloader.TestLoader(test_paths, test_types,
                                        testloader.TestFilter(), run_info)

    for test_type, tests in test_loader.disabled_tests.iteritems():
        for test in tests:
            rv.append({"test": test.id, "reason": test.disabled()})
    print json.dumps(rv, indent=2)
Пример #4
0
def list_test_groups(serve_root, test_paths, test_types, product, **kwargs):

    do_delayed_imports(serve_root)

    run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
    test_filter = testloader.TestFilter(
        include=kwargs["include"],
        exclude=kwargs["exclude"],
        manifest_path=kwargs["include_manifest"])
    test_loader = testloader.TestLoader(test_paths, test_types, test_filter,
                                        run_info)

    for item in sorted(test_loader.groups(test_types)):
        print item
Пример #5
0
def get_loader(test_paths,
               product,
               debug=None,
               run_info_extras=None,
               **kwargs):
    if run_info_extras is None:
        run_info_extras = {}

    run_info = wpttest.get_run_info(
        kwargs["run_info"],
        product,
        browser_version=kwargs.get("browser_version"),
        browser_channel=kwargs.get("browser_channel"),
        verify=kwargs.get("verify"),
        debug=debug,
        extras=run_info_extras,
        enable_webrender=kwargs.get("enable_webrender"))

    test_manifests = testloader.ManifestLoader(
        test_paths,
        force_manifest_update=kwargs["manifest_update"],
        manifest_download=kwargs["manifest_download"]).load()

    manifest_filters = []

    if kwargs["include"] or kwargs["exclude"] or kwargs[
            "include_manifest"] or kwargs["default_exclude"]:
        manifest_filters.append(
            testloader.TestFilter(include=kwargs["include"],
                                  exclude=kwargs["exclude"],
                                  manifest_path=kwargs["include_manifest"],
                                  test_manifests=test_manifests,
                                  explicit=kwargs["default_exclude"]))

    ssl_enabled = sslutils.get_cls(kwargs["ssl_type"]).ssl_enabled
    test_loader = testloader.TestLoader(test_manifests,
                                        kwargs["test_types"],
                                        run_info,
                                        manifest_filters=manifest_filters,
                                        chunk_type=kwargs["chunk_type"],
                                        total_chunks=kwargs["total_chunks"],
                                        chunk_number=kwargs["this_chunk"],
                                        include_https=ssl_enabled,
                                        skip_timeout=kwargs["skip_timeout"])
    return run_info, test_loader
Пример #6
0
def get_loader(test_paths, product, ssl_env, debug=False, **kwargs):
    run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)

    test_manifests = testloader.ManifestLoader(
        test_paths, force_manifest_update=kwargs["manifest_update"]).load()

    test_filter = testloader.TestFilter(
        include=kwargs["include"],
        exclude=kwargs["exclude"],
        manifest_path=kwargs["include_manifest"],
        test_manifests=test_manifests)

    test_loader = testloader.TestLoader(test_manifests,
                                        kwargs["test_types"],
                                        test_filter,
                                        run_info,
                                        chunk_type=kwargs["chunk_type"],
                                        total_chunks=kwargs["total_chunks"],
                                        chunk_number=kwargs["this_chunk"],
                                        include_https=ssl_env.ssl_enabled)
    return run_info, test_loader
Пример #7
0
def run_tests(config, serve_root, test_paths, product, **kwargs):
    logging_queue = None
    logging_thread = None
    original_stdio = (sys.stdout, sys.stderr)
    test_queues = None

    try:
        if not kwargs["no_capture_stdio"]:
            logging_queue = Queue()
            logging_thread = LogThread(logging_queue, logger, "info")
            sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT")
            sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR")
            logging_thread.start()

        do_delayed_imports(serve_root)

        run_info = wpttest.get_run_info(kwargs["run_info"],
                                        product,
                                        debug=False)

        (check_args, browser_cls, get_browser_kwargs, executor_classes,
         get_executor_kwargs,
         env_options) = products.load_product(config, product)

        check_args(**kwargs)

        ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
        ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))

        unexpected_total = 0

        if "test_loader" in kwargs:
            test_loader = kwargs["test_loader"]
        else:
            test_filter = testloader.TestFilter(
                include=kwargs["include"],
                exclude=kwargs["exclude"],
                manifest_path=kwargs["include_manifest"])
            test_loader = testloader.TestLoader(
                test_paths, kwargs["test_types"], test_filter, run_info,
                kwargs["chunk_type"], kwargs["total_chunks"],
                kwargs["this_chunk"], kwargs["manifest_update"])

        if kwargs["run_by_dir"] is False:
            test_source_cls = testloader.SingleTestSource
            test_source_kwargs = {}
        else:
            # A value of None indicates infinite depth
            test_source_cls = testloader.PathGroupedSource
            test_source_kwargs = {"depth": kwargs["run_by_dir"]}

        logger.info("Using %i client processes" % kwargs["processes"])

        with TestEnvironment(serve_root, test_paths, ssl_env,
                             env_options) as test_environment:
            try:
                test_environment.ensure_started()
            except TestEnvironmentError as e:
                logger.critical("Error starting test environment: %s" %
                                e.message)
                raise

            browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
            base_server = "http://%s:%i" % (
                test_environment.external_config["host"],
                test_environment.external_config["ports"]["http"][0])

            repeat = kwargs["repeat"]
            for repeat_count in xrange(repeat):
                if repeat > 1:
                    logger.info("Repetition %i / %i" %
                                (repeat_count + 1, repeat))

                unexpected_count = 0
                logger.suite_start(test_loader.test_ids, run_info)
                for test_type in kwargs["test_types"]:
                    logger.info("Running %s tests" % test_type)

                    for test in test_loader.disabled_tests[test_type]:
                        logger.test_start(test.id)
                        logger.test_end(test.id, status="SKIP")

                    executor_cls = executor_classes.get(test_type)
                    executor_kwargs = get_executor_kwargs(
                        base_server, **kwargs)

                    if executor_cls is None:
                        logger.error(
                            "Unsupported test type %s for product %s" %
                            (test_type, product))
                        continue

                    with ManagerGroup(
                            "web-platform-tests", kwargs["processes"],
                            test_source_cls, test_source_kwargs, browser_cls,
                            browser_kwargs, executor_cls, executor_kwargs,
                            kwargs["pause_on_unexpected"]) as manager_group:
                        try:
                            manager_group.run(test_type, test_loader.tests)
                        except KeyboardInterrupt:
                            logger.critical("Main thread got signal")
                            manager_group.stop()
                            raise
                    unexpected_count += manager_group.unexpected_count()

                unexpected_total += unexpected_count
                logger.info("Got %i unexpected results" % unexpected_count)
                logger.suite_end()
    except KeyboardInterrupt:
        if test_queues is not None:
            for queue in test_queues.itervalues():
                queue.cancel_join_thread()
    finally:
        if test_queues is not None:
            for queue in test_queues.itervalues():
                queue.close()
        sys.stdout, sys.stderr = original_stdio
        if not kwargs["no_capture_stdio"] and logging_queue is not None:
            logger.info("Closing logging queue")
            logging_queue.put(None)
            if logging_thread is not None:
                logging_thread.join(10)
            logging_queue.close()

    return manager_group.unexpected_count() == 0