def get_loader(test_paths, product, ssl_env, debug=None, **kwargs): run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug) test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"]).load() manifest_filters = [] meta_filters = [] if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]: manifest_filters.append( testloader.TestFilter( include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"], test_manifests=test_manifests, ) ) if kwargs["tags"]: meta_filters.append(testloader.TagFilter(tags=kwargs["tags"])) test_loader = testloader.TestLoader( test_manifests, kwargs["test_types"], run_info, manifest_filters=manifest_filters, meta_filters=meta_filters, chunk_type=kwargs["chunk_type"], total_chunks=kwargs["total_chunks"], chunk_number=kwargs["this_chunk"], include_https=ssl_env.ssl_enabled, ) return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=None, run_info_extras=None, **kwargs): if run_info_extras is None: run_info_extras = {} run_info = wpttest.get_run_info(kwargs["run_info"], product, browser_version=kwargs.get("browser_version"), debug=debug, extras=run_info_extras) test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"], manifest_download=kwargs["manifest_download"]).load() manifest_filters = [] meta_filters = [] if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]: manifest_filters.append(testloader.TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"], test_manifests=test_manifests)) if kwargs["tags"]: meta_filters.append(testloader.TagFilter(tags=kwargs["tags"])) test_loader = testloader.TestLoader(test_manifests, kwargs["test_types"], run_info, manifest_filters=manifest_filters, meta_filters=meta_filters, chunk_type=kwargs["chunk_type"], total_chunks=kwargs["total_chunks"], chunk_number=kwargs["this_chunk"], include_https=ssl_env.ssl_enabled, skip_timeout=kwargs["skip_timeout"]) return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=None, **kwargs): run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug) test_manifests = testloader.ManifestLoader( test_paths, force_manifest_update=kwargs["manifest_update"]).load() manifest_filters = [] meta_filters = [] if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]: manifest_filters.append( testloader.TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"], test_manifests=test_manifests)) if kwargs["tags"]: meta_filters.append(testloader.TagFilter(tags=kwargs["tags"])) test_loader = testloader.TestLoader(test_manifests, kwargs["test_types"], run_info, manifest_filters=manifest_filters, meta_filters=meta_filters, chunk_type=kwargs["chunk_type"], total_chunks=kwargs["total_chunks"], chunk_number=kwargs["this_chunk"], include_https=ssl_env.ssl_enabled) return run_info, test_loader
def list_disabled(tests_root, metadata_root, test_types, product, **kwargs): rv = [] run_info = wpttest.get_run_info(metadata_root, product, debug=False) test_loader = TestLoader(tests_root, metadata_root, TestFilter(), run_info) for test_type, tests in test_loader.get_disabled(test_types).iteritems(): for test in tests: rv.append({"test": test.id, "reason": test.disabled()}) print json.dumps(rv, indent=2)
def list_test_groups(tests_root, metadata_root, test_types, product, **kwargs): do_test_relative_imports(tests_root) run_info = wpttest.get_run_info(metadata_root, product, debug=False) test_filter = TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"]) test_loader = TestLoader(tests_root, metadata_root, test_filter, run_info) for item in sorted(test_loader.get_groups(test_types)): print item
def list_disabled(serve_root, test_paths, test_types, product, **kwargs): do_delayed_imports(serve_root) rv = [] run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False) test_loader = testloader.TestLoader(test_paths, test_types, testloader.TestFilter(), run_info) for test_type, tests in test_loader.disabled_tests.iteritems(): for test in tests: rv.append({"test": test.id, "reason": test.disabled()}) print json.dumps(rv, indent=2)
def list_disabled(tests_root, metadata_root, test_types, product, **kwargs): do_test_relative_imports(tests_root) rv = [] run_info = wpttest.get_run_info(metadata_root, product, debug=False) test_loader = TestLoader(tests_root, metadata_root, test_types, TestFilter(), run_info) for test_type, tests in test_loader.disabled_tests.iteritems(): for test in tests: rv.append({"test": test.id, "reason": test.disabled()}) print json.dumps(rv, indent=2)
def list_test_groups(tests_root, metadata_root, test_types, product, **kwargs): do_test_relative_imports(tests_root) run_info = wpttest.get_run_info(metadata_root, product, debug=False) test_filter = TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"]) test_loader = TestLoader(tests_root, metadata_root, test_types, test_filter, run_info) for item in sorted(test_loader.groups()): print item
def list_test_groups(serve_root, test_paths, test_types, product, **kwargs): do_delayed_imports(serve_root) run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False) test_filter = testloader.TestFilter( include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"]) test_loader = testloader.TestLoader(test_paths, test_types, test_filter, run_info) for item in sorted(test_loader.groups(test_types)): print item
def __init__(self, target, **kwargs): self.target = target self.test_type = kwargs["test_types"][0] run_info = wpttest.get_run_info(kwargs["product"], debug=False) test_filter = wptrunner.TestFilter(include=kwargs["include"]) self.test_loader = wptrunner.TestLoader(kwargs["tests_root"], kwargs["metadata_root"], test_filter, run_info) if kwargs["repeat"] == 1: logger.critical("Need to specify --repeat with more than one repetition") sys.exit(1) self.kwargs = kwargs
def list_test_groups(serve_root, test_paths, test_types, product, **kwargs): do_delayed_imports(serve_root) run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False) test_filter = testloader.TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"]) test_loader = testloader.TestLoader(test_paths, test_types, test_filter, run_info) for item in sorted(test_loader.groups()): print item
def __init__(self, target, **kwargs): self.target = target self.test_type = kwargs["test_types"][0] run_info = wpttest.get_run_info(kwargs["metadata_root"], kwargs["product"], debug=False) test_filter = wptrunner.TestFilter(include=kwargs["include"]) self.test_loader = wptrunner.TestLoader(kwargs["tests_root"], kwargs["metadata_root"], test_filter, run_info) if kwargs["repeat"] == 1: logger.critical( "Need to specify --repeat with more than one repetition") sys.exit(1) self.kwargs = kwargs
def get_loader(test_paths, product, debug=None, run_info_extras=None, **kwargs): if run_info_extras is None: run_info_extras = {} run_info = wpttest.get_run_info( kwargs["run_info"], product, browser_version=kwargs.get("browser_version"), browser_channel=kwargs.get("browser_channel"), verify=kwargs.get("verify"), debug=debug, extras=run_info_extras, enable_webrender=kwargs.get("enable_webrender")) test_manifests = testloader.ManifestLoader( test_paths, force_manifest_update=kwargs["manifest_update"], manifest_download=kwargs["manifest_download"]).load() manifest_filters = [] if kwargs["include"] or kwargs["exclude"] or kwargs[ "include_manifest"] or kwargs["default_exclude"]: manifest_filters.append( testloader.TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"], test_manifests=test_manifests, explicit=kwargs["default_exclude"])) ssl_enabled = sslutils.get_cls(kwargs["ssl_type"]).ssl_enabled test_loader = testloader.TestLoader(test_manifests, kwargs["test_types"], run_info, manifest_filters=manifest_filters, chunk_type=kwargs["chunk_type"], total_chunks=kwargs["total_chunks"], chunk_number=kwargs["this_chunk"], include_https=ssl_enabled, skip_timeout=kwargs["skip_timeout"]) return run_info, test_loader
def get_loader(test_paths, product, debug=False, **kwargs): run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug) test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"]).load() test_filter = testloader.TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"], test_manifests=test_manifests) test_loader = testloader.TestLoader(test_manifests, kwargs["test_types"], test_filter, run_info, chunk_type=kwargs["chunk_type"], total_chunks=kwargs["total_chunks"], chunk_number=kwargs["this_chunk"]) return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=False, **kwargs): run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug) test_manifests = testloader.ManifestLoader( test_paths, force_manifest_update=kwargs["manifest_update"]).load() test_filter = testloader.TestFilter( include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"], test_manifests=test_manifests) test_loader = testloader.TestLoader(test_manifests, kwargs["test_types"], test_filter, run_info, chunk_type=kwargs["chunk_type"], total_chunks=kwargs["total_chunks"], chunk_number=kwargs["this_chunk"], include_https=ssl_env.ssl_enabled) return run_info, test_loader
def run_tests(config, test_paths, product, **kwargs): with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]): env.do_delayed_imports(logger, test_paths) (check_args, target_browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options, get_env_extras, run_info_extras) = products.load_product(config, product) ssl_env = env.ssl_env(logger, **kwargs) env_extras = get_env_extras(**kwargs) check_args(**kwargs) if kwargs["install_fonts"]: env_extras.append( FontInstaller(font_dir=kwargs["font_dir"], ahem=os.path.join(kwargs["tests_root"], "fonts/Ahem.ttf"))) if "test_loader" in kwargs: run_info = wpttest.get_run_info( kwargs["run_info"], product, browser_version=kwargs.get("browser_version"), debug=None, extras=run_info_extras(**kwargs)) test_loader = kwargs["test_loader"] else: run_info, test_loader = get_loader( test_paths, product, ssl_env, run_info_extras=run_info_extras(**kwargs), **kwargs) test_source_kwargs = {"processes": kwargs["processes"]} if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs["depth"] = kwargs["run_by_dir"] logger.info("Using %i client processes" % kwargs["processes"]) test_total = 0 unexpected_total = 0 kwargs["pause_after_test"] = get_pause_after_test( test_loader, **kwargs) with env.TestEnvironment(test_paths, ssl_env, kwargs["pause_after_test"], kwargs["debug_info"], env_options, env_extras) as test_environment: try: test_environment.ensure_started() except env.TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise repeat = kwargs["repeat"] repeat_count = 0 repeat_until_unexpected = kwargs["repeat_until_unexpected"] while repeat_count < repeat or repeat_until_unexpected: repeat_count += 1 if repeat_until_unexpected: logger.info("Repetition %i" % (repeat_count)) elif repeat > 1: logger.info("Repetition %i / %i" % (repeat_count, repeat)) test_count = 0 unexpected_count = 0 logger.suite_start(test_loader.test_ids, name='web-platform-test', run_info=run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) # WebDriver tests may create and destroy multiple browser # processes as part of their expected behavior. These # processes are managed by a WebDriver server binary. This # obviates the need for wptrunner to provide a browser, so # the NullBrowser is used in place of the "target" browser if test_type == "wdspec": browser_cls = NullBrowser else: browser_cls = target_browser_cls browser_kwargs = get_browser_kwargs( test_type, run_info, ssl_env=ssl_env, config=test_environment.config, **kwargs) executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs( test_type, test_environment.config, test_environment.cache_manager, run_info, **kwargs) if executor_cls is None: logger.error( "Unsupported test type %s for product %s" % (test_type, product)) continue for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") if test_type == "testharness": run_tests = {"testharness": []} for test in test_loader.tests["testharness"]: if test.testdriver and not executor_cls.supports_testdriver: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") elif test.jsshell and not executor_cls.supports_jsshell: # We expect that tests for JavaScript shells # will not be run along with tests that run in # a full web browser, so we silently skip them # here. pass else: run_tests["testharness"].append(test) else: run_tests = test_loader.tests with ManagerGroup("web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["rerun"], kwargs["pause_after_test"], kwargs["pause_on_unexpected"], kwargs["restart_on_unexpected"], kwargs["debug_info"]) as manager_group: try: manager_group.run(test_type, run_tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise test_count += manager_group.test_count() unexpected_count += manager_group.unexpected_count() test_total += test_count unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) if repeat_until_unexpected and unexpected_total > 0: break logger.suite_end() if test_total == 0: logger.error("No tests ran") return False if unexpected_total and not kwargs["fail_on_unexpected"]: logger.info("Tolerating %s unexpected results" % unexpected_total) return True return unexpected_total == 0
def run_tests(config, test_paths, product, **kwargs): with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]): env.do_delayed_imports(logger, test_paths) (check_args, target_browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options, get_env_extras, run_info_extras) = products.load_product(config, product) ssl_env = env.ssl_env(logger, **kwargs) env_extras = get_env_extras(**kwargs) check_args(**kwargs) if kwargs["install_fonts"]: env_extras.append(FontInstaller( font_dir=kwargs["font_dir"], ahem=os.path.join(kwargs["tests_root"], "fonts/Ahem.ttf") )) if "test_loader" in kwargs: run_info = wpttest.get_run_info(kwargs["run_info"], product, browser_version=kwargs.get("browser_version"), debug=None, extras=run_info_extras(**kwargs)) test_loader = kwargs["test_loader"] else: run_info, test_loader = get_loader(test_paths, product, ssl_env, run_info_extras=run_info_extras(**kwargs), **kwargs) test_source_kwargs = {"processes": kwargs["processes"]} if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs["depth"] = kwargs["run_by_dir"] logger.info("Using %i client processes" % kwargs["processes"]) test_total = 0 unexpected_total = 0 kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs) with env.TestEnvironment(test_paths, ssl_env, kwargs["pause_after_test"], kwargs["debug_info"], env_options, env_extras) as test_environment: try: test_environment.ensure_started() except env.TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise repeat = kwargs["repeat"] repeat_count = 0 repeat_until_unexpected = kwargs["repeat_until_unexpected"] while repeat_count < repeat or repeat_until_unexpected: repeat_count += 1 if repeat_until_unexpected: logger.info("Repetition %i" % (repeat_count)) elif repeat > 1: logger.info("Repetition %i / %i" % (repeat_count, repeat)) test_count = 0 unexpected_count = 0 logger.suite_start(test_loader.test_ids, name='web-platform-test', run_info=run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) # WebDriver tests may create and destroy multiple browser # processes as part of their expected behavior. These # processes are managed by a WebDriver server binary. This # obviates the need for wptrunner to provide a browser, so # the NullBrowser is used in place of the "target" browser if test_type == "wdspec": browser_cls = NullBrowser else: browser_cls = target_browser_cls browser_kwargs = get_browser_kwargs(test_type, run_info, ssl_env=ssl_env, config=test_environment.config, **kwargs) executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs(test_type, test_environment.config, test_environment.cache_manager, run_info, **kwargs) if executor_cls is None: logger.error("Unsupported test type %s for product %s" % (test_type, product)) continue for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") if test_type == "testharness": run_tests = {"testharness": []} for test in test_loader.tests["testharness"]: if test.testdriver and not executor_cls.supports_testdriver: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") elif test.jsshell and not executor_cls.supports_jsshell: # We expect that tests for JavaScript shells # will not be run along with tests that run in # a full web browser, so we silently skip them # here. pass else: run_tests["testharness"].append(test) else: run_tests = test_loader.tests with ManagerGroup("web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["rerun"], kwargs["pause_after_test"], kwargs["pause_on_unexpected"], kwargs["restart_on_unexpected"], kwargs["debug_info"]) as manager_group: try: manager_group.run(test_type, run_tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise test_count += manager_group.test_count() unexpected_count += manager_group.unexpected_count() test_total += test_count unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) if repeat_until_unexpected and unexpected_total > 0: break logger.suite_end() if test_total == 0: logger.error("No tests ran") return False if unexpected_total and not kwargs["fail_on_unexpected"]: logger.info("Tolerating %s unexpected results" % unexpected_total) return True return unexpected_total == 0
def run_tests(config, tests_root, metadata_root, product, **kwargs): logging_queue = None logging_thread = None original_stdio = (sys.stdout, sys.stderr) test_queues = None try: if not kwargs["no_capture_stdio"]: logging_queue = Queue() logging_thread = LogThread(logging_queue, logger, "info") sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT") sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR") logging_thread.start() do_test_relative_imports(tests_root) run_info = wpttest.get_run_info(metadata_root, product, debug=False) (check_args, browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options) = products.load_product(config, product) check_args(**kwargs) browser_kwargs = get_browser_kwargs(**kwargs) unexpected_total = 0 if "test_loader" in kwargs: test_loader = kwargs["test_loader"] else: test_filter = TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"]) test_loader = TestLoader(tests_root, metadata_root, test_filter, run_info) logger.info("Using %i client processes" % kwargs["processes"]) with TestEnvironment(tests_root, env_options) as test_environment: try: test_environment.ensure_started() except TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise base_server = "http://%s:%i" % (test_environment.config["host"], test_environment.config["ports"]["http"][0]) repeat = kwargs["repeat"] for repeat_count in xrange(repeat): if repeat > 1: logger.info("Repetition %i / %i" % (repeat_count + 1, repeat)) test_ids, test_queues = test_loader.queue_tests(kwargs["test_types"], kwargs["chunk_type"], kwargs["total_chunks"], kwargs["this_chunk"]) unexpected_count = 0 logger.suite_start(test_ids, run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) tests_queue = test_queues[test_type] executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs(base_server, **kwargs) if executor_cls is None: logger.error("Unsupported test type %s for product %s" % (test_type, product)) continue with ManagerGroup("web-platform-tests", kwargs["processes"], browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["pause_on_unexpected"]) as manager_group: try: manager_group.start(tests_queue) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise manager_group.wait() unexpected_count += manager_group.unexpected_count() unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) logger.suite_end() except KeyboardInterrupt: if test_queues is not None: for queue in test_queues.itervalues(): queue.cancel_join_thread() finally: if test_queues is not None: for queue in test_queues.itervalues(): queue.close() sys.stdout, sys.stderr = original_stdio if not kwargs["no_capture_stdio"] and logging_queue is not None: logger.info("Closing logging queue") logging_queue.put(None) if logging_thread is not None: logging_thread.join(10) logging_queue.close() return manager_group.unexpected_count() == 0
def run_tests(config, test_paths, product, **kwargs): with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]): env.do_delayed_imports(logger, test_paths) (check_args, browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options) = products.load_product(config, product) ssl_env = env.ssl_env(logger, **kwargs) check_args(**kwargs) if "test_loader" in kwargs: run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False) test_loader = kwargs["test_loader"] else: run_info, test_loader = get_loader(test_paths, product, ssl_env, **kwargs) if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource test_source_kwargs = {} else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs = {"depth": kwargs["run_by_dir"]} logger.info("Using %i client processes" % kwargs["processes"]) unexpected_total = 0 kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs) with env.TestEnvironment(test_paths, ssl_env, kwargs["pause_after_test"], env_options) as test_environment: try: test_environment.ensure_started() except env.TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs) repeat = kwargs["repeat"] for repeat_count in xrange(repeat): if repeat > 1: logger.info("Repetition %i / %i" % (repeat_count + 1, repeat)) unexpected_count = 0 logger.suite_start(test_loader.test_ids, run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs(test_type, test_environment.external_config, test_environment.cache_manager, **kwargs) if executor_cls is None: logger.error("Unsupported test type %s for product %s" % (test_type, product)) continue with ManagerGroup("web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["pause_after_test"], kwargs["pause_on_unexpected"], kwargs["debug_args"]) as manager_group: try: manager_group.run(test_type, test_loader.tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise unexpected_count += manager_group.unexpected_count() unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) logger.suite_end() return unexpected_total == 0
def run_tests(config, test_paths, product, **kwargs): with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]): env.do_delayed_imports(logger, test_paths) (check_args, browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options, run_info_extras) = products.load_product(config, product) ssl_env = env.ssl_env(logger, **kwargs) check_args(**kwargs) if "test_loader" in kwargs: run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=None, extras=run_info_extras(**kwargs)) test_loader = kwargs["test_loader"] else: run_info, test_loader = get_loader(test_paths, product, ssl_env, run_info_extras=run_info_extras(**kwargs), **kwargs) if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource test_source_kwargs = {} else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs = {"depth": kwargs["run_by_dir"]} logger.info("Using %i client processes" % kwargs["processes"]) unexpected_total = 0 kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs) with env.TestEnvironment(test_paths, ssl_env, kwargs["pause_after_test"], kwargs["debug_info"], env_options) as test_environment: try: test_environment.ensure_started() except env.TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs) repeat = kwargs["repeat"] for repeat_count in xrange(repeat): if repeat > 1: logger.info("Repetition %i / %i" % (repeat_count + 1, repeat)) unexpected_count = 0 logger.suite_start(test_loader.test_ids, run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs(test_type, test_environment.external_config, test_environment.cache_manager, run_info, **kwargs) if executor_cls is None: logger.error("Unsupported test type %s for product %s" % (test_type, product)) continue with ManagerGroup("web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["pause_after_test"], kwargs["pause_on_unexpected"], kwargs["debug_info"]) as manager_group: try: manager_group.run(test_type, test_loader.tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise unexpected_count += manager_group.unexpected_count() unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) logger.suite_end() return unexpected_total == 0
def run_tests(config, test_paths, product, **kwargs): with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]): env.do_delayed_imports(logger, test_paths) (check_args, target_browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options, run_info_extras) = products.load_product(config, product) ssl_env = env.ssl_env(logger, **kwargs) check_args(**kwargs) if "test_loader" in kwargs: run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=None, extras=run_info_extras(**kwargs)) test_loader = kwargs["test_loader"] else: run_info, test_loader = get_loader( test_paths, product, ssl_env, run_info_extras=run_info_extras(**kwargs), **kwargs) if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource test_source_kwargs = {} else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs = {"depth": kwargs["run_by_dir"]} logger.info("Using %i client processes" % kwargs["processes"]) unexpected_total = 0 kwargs["pause_after_test"] = get_pause_after_test( test_loader, **kwargs) with env.TestEnvironment(test_paths, ssl_env, kwargs["pause_after_test"], kwargs["debug_info"], env_options) as test_environment: try: test_environment.ensure_started() except env.TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise repeat = kwargs["repeat"] repeat_count = 0 repeat_until_unexpected = kwargs["repeat_until_unexpected"] while repeat_count < repeat or repeat_until_unexpected: repeat_count += 1 if repeat_until_unexpected: logger.info("Repetition %i" % (repeat_count)) elif repeat > 1: logger.info("Repetition %i / %i" % (repeat_count, repeat)) unexpected_count = 0 logger.suite_start(test_loader.test_ids, run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) # WebDriver tests may create and destroy multiple browser # processes as part of their expected behavior. These # processes are managed by a WebDriver server binary. This # obviates the need for wptrunner to provide a browser, so # the NullBrowser is used in place of the "target" browser if test_type == "wdspec": browser_cls = NullBrowser else: browser_cls = target_browser_cls browser_kwargs = get_browser_kwargs(test_type, run_info, ssl_env=ssl_env, **kwargs) executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs( test_type, test_environment.external_config, test_environment.cache_manager, run_info, **kwargs) if executor_cls is None: logger.error( "Unsupported test type %s for product %s" % (test_type, product)) continue for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") with ManagerGroup("web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["pause_after_test"], kwargs["pause_on_unexpected"], kwargs["restart_on_unexpected"], kwargs["debug_info"]) as manager_group: try: manager_group.run(test_type, test_loader.tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise unexpected_count += manager_group.unexpected_count() unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) if repeat_until_unexpected and unexpected_total > 0: break logger.suite_end() return unexpected_total == 0
def run_tests(config, serve_root, test_paths, product, **kwargs): logging_queue = None logging_thread = None original_stdio = (sys.stdout, sys.stderr) test_queues = None try: if not kwargs["no_capture_stdio"]: logging_queue = Queue() logging_thread = LogThread(logging_queue, logger, "info") sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT") sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR") logging_thread.start() do_delayed_imports(serve_root) run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False) (check_args, browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options) = products.load_product(config, product) check_args(**kwargs) ssl_env_cls = sslutils.environments[kwargs["ssl_type"]] ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs)) unexpected_total = 0 if "test_loader" in kwargs: test_loader = kwargs["test_loader"] else: test_filter = testloader.TestFilter( include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"]) test_loader = testloader.TestLoader( test_paths, kwargs["test_types"], test_filter, run_info, kwargs["chunk_type"], kwargs["total_chunks"], kwargs["this_chunk"], kwargs["manifest_update"]) if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource test_source_kwargs = {} else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs = {"depth": kwargs["run_by_dir"]} logger.info("Using %i client processes" % kwargs["processes"]) with TestEnvironment(serve_root, test_paths, ssl_env, env_options) as test_environment: try: test_environment.ensure_started() except TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs) base_server = "http://%s:%i" % ( test_environment.external_config["host"], test_environment.external_config["ports"]["http"][0]) repeat = kwargs["repeat"] for repeat_count in xrange(repeat): if repeat > 1: logger.info("Repetition %i / %i" % (repeat_count + 1, repeat)) unexpected_count = 0 logger.suite_start(test_loader.test_ids, run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs( base_server, **kwargs) if executor_cls is None: logger.error( "Unsupported test type %s for product %s" % (test_type, product)) continue with ManagerGroup( "web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["pause_on_unexpected"]) as manager_group: try: manager_group.run(test_type, test_loader.tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise unexpected_count += manager_group.unexpected_count() unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) logger.suite_end() except KeyboardInterrupt: if test_queues is not None: for queue in test_queues.itervalues(): queue.cancel_join_thread() finally: if test_queues is not None: for queue in test_queues.itervalues(): queue.close() sys.stdout, sys.stderr = original_stdio if not kwargs["no_capture_stdio"] and logging_queue is not None: logger.info("Closing logging queue") logging_queue.put(None) if logging_thread is not None: logging_thread.join(10) logging_queue.close() return manager_group.unexpected_count() == 0
def run_tests(config, test_paths, product, **kwargs): with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]): env.do_delayed_imports(logger, test_paths) (check_args, target_browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options, get_env_extras, run_info_extras) = products.load_product(config, product) ssl_env = env.ssl_env(logger, **kwargs) env_extras = get_env_extras(**kwargs) check_args(**kwargs) if "test_loader" in kwargs: run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=None, extras=run_info_extras(**kwargs)) test_loader = kwargs["test_loader"] else: run_info, test_loader = get_loader(test_paths, product, ssl_env, run_info_extras=run_info_extras(**kwargs), **kwargs) if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource test_source_kwargs = {} else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs = {"depth": kwargs["run_by_dir"]} logger.info("Using %i client processes" % kwargs["processes"]) unexpected_total = 0 kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs) with env.TestEnvironment(test_paths, ssl_env, kwargs["pause_after_test"], kwargs["debug_info"], env_options, env_extras) as test_environment: try: test_environment.ensure_started() except env.TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs) repeat = kwargs["repeat"] repeat_count = 0 repeat_until_unexpected = kwargs["repeat_until_unexpected"] while repeat_count < repeat or repeat_until_unexpected: repeat_count += 1 if repeat_until_unexpected: logger.info("Repetition %i" % (repeat_count)) elif repeat > 1: logger.info("Repetition %i / %i" % (repeat_count, repeat)) unexpected_count = 0 logger.suite_start(test_loader.test_ids, run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) # WebDriver tests may create and destroy multiple browser # processes as part of their expected behavior. These # processes are managed by a WebDriver server binary. This # obviates the need for wptrunner to provide a browser, so # the NullBrowser is used in place of the "target" browser if test_type == "wdspec": browser_cls = NullBrowser else: browser_cls = target_browser_cls for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs(test_type, test_environment.external_config, test_environment.cache_manager, run_info, **kwargs) if executor_cls is None: logger.error("Unsupported test type %s for product %s" % (test_type, product)) continue with ManagerGroup("web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["pause_after_test"], kwargs["pause_on_unexpected"], kwargs["restart_on_unexpected"], kwargs["debug_info"]) as manager_group: try: manager_group.run(test_type, test_loader.tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise unexpected_count += manager_group.unexpected_count() unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) if repeat_until_unexpected and unexpected_total > 0: break logger.suite_end() return unexpected_total == 0
def run_tests(config, serve_root, test_paths, product, **kwargs): logging_queue = None logging_thread = None original_stdio = (sys.stdout, sys.stderr) test_queues = None try: if not kwargs["no_capture_stdio"]: logging_queue = Queue() logging_thread = LogThread(logging_queue, logger, "info") sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT") sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR") logging_thread.start() do_delayed_imports(serve_root) run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False) (check_args, browser_cls, get_browser_kwargs, executor_classes, get_executor_kwargs, env_options) = products.load_product(config, product) check_args(**kwargs) ssl_env_cls = sslutils.environments[kwargs["ssl_type"]] ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs)) unexpected_total = 0 if "test_loader" in kwargs: test_loader = kwargs["test_loader"] else: test_filter = testloader.TestFilter(include=kwargs["include"], exclude=kwargs["exclude"], manifest_path=kwargs["include_manifest"]) test_loader = testloader.TestLoader(test_paths, kwargs["test_types"], test_filter, run_info, kwargs["chunk_type"], kwargs["total_chunks"], kwargs["this_chunk"], kwargs["manifest_update"]) if kwargs["run_by_dir"] is False: test_source_cls = testloader.SingleTestSource test_source_kwargs = {} else: # A value of None indicates infinite depth test_source_cls = testloader.PathGroupedSource test_source_kwargs = {"depth": kwargs["run_by_dir"]} logger.info("Using %i client processes" % kwargs["processes"]) with TestEnvironment(serve_root, test_paths, ssl_env, env_options) as test_environment: try: test_environment.ensure_started() except TestEnvironmentError as e: logger.critical("Error starting test environment: %s" % e.message) raise browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs) base_server = "http://%s:%i" % (test_environment.external_config["host"], test_environment.external_config["ports"]["http"][0]) repeat = kwargs["repeat"] for repeat_count in xrange(repeat): if repeat > 1: logger.info("Repetition %i / %i" % (repeat_count + 1, repeat)) unexpected_count = 0 logger.suite_start(test_loader.test_ids, run_info) for test_type in kwargs["test_types"]: logger.info("Running %s tests" % test_type) for test in test_loader.disabled_tests[test_type]: logger.test_start(test.id) logger.test_end(test.id, status="SKIP") executor_cls = executor_classes.get(test_type) executor_kwargs = get_executor_kwargs(base_server, **kwargs) if executor_cls is None: logger.error("Unsupported test type %s for product %s" % (test_type, product)) continue with ManagerGroup("web-platform-tests", kwargs["processes"], test_source_cls, test_source_kwargs, browser_cls, browser_kwargs, executor_cls, executor_kwargs, kwargs["pause_on_unexpected"], kwargs["debug_args"]) as manager_group: try: manager_group.run(test_type, test_loader.tests) except KeyboardInterrupt: logger.critical("Main thread got signal") manager_group.stop() raise unexpected_count += manager_group.unexpected_count() unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) logger.suite_end() except KeyboardInterrupt: if test_queues is not None: for queue in test_queues.itervalues(): queue.cancel_join_thread() finally: if test_queues is not None: for queue in test_queues.itervalues(): queue.close() sys.stdout, sys.stderr = original_stdio if not kwargs["no_capture_stdio"] and logging_queue is not None: logger.info("Closing logging queue") logging_queue.put(None) if logging_thread is not None: logging_thread.join(10) logging_queue.close() return unexpected_total == 0