def get_loader(test_paths, product, ssl_env, debug=None, **kwargs):
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"]).load()
manifest_filters = []
meta_filters = []
if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
manifest_filters.append(
testloader.TestFilter(
include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests,
)
)
if kwargs["tags"]:
meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))
test_loader = testloader.TestLoader(
test_manifests,
kwargs["test_types"],
run_info,
manifest_filters=manifest_filters,
meta_filters=meta_filters,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_env.ssl_enabled,
)
return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=None, run_info_extras=None, **kwargs):
if run_info_extras is None:
run_info_extras = {}
run_info = wpttest.get_run_info(kwargs["run_info"], product,
browser_version=kwargs.get("browser_version"),
debug=debug,
extras=run_info_extras)
test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"],
manifest_download=kwargs["manifest_download"]).load()
manifest_filters = []
meta_filters = []
if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
manifest_filters.append(testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests))
if kwargs["tags"]:
meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
run_info,
manifest_filters=manifest_filters,
meta_filters=meta_filters,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_env.ssl_enabled,
skip_timeout=kwargs["skip_timeout"])
return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=None, **kwargs):
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
test_manifests = testloader.ManifestLoader(
test_paths, force_manifest_update=kwargs["manifest_update"]).load()
manifest_filters = []
meta_filters = []
if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
manifest_filters.append(
testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests))
if kwargs["tags"]:
meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
run_info,
manifest_filters=manifest_filters,
meta_filters=meta_filters,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_env.ssl_enabled)
return run_info, test_loader
def list_disabled(tests_root, metadata_root, test_types, product, **kwargs):
rv = []
run_info = wpttest.get_run_info(metadata_root, product, debug=False)
test_loader = TestLoader(tests_root, metadata_root, TestFilter(), run_info)
for test_type, tests in test_loader.get_disabled(test_types).iteritems():
for test in tests:
rv.append({"test": test.id, "reason": test.disabled()})
print json.dumps(rv, indent=2)
def list_test_groups(tests_root, metadata_root, test_types, product, **kwargs):
do_test_relative_imports(tests_root)
run_info = wpttest.get_run_info(metadata_root, product, debug=False)
test_filter = TestFilter(include=kwargs["include"], exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = TestLoader(tests_root, metadata_root, test_filter, run_info)
for item in sorted(test_loader.get_groups(test_types)):
print item
def list_disabled(serve_root, test_paths, test_types, product, **kwargs):
do_delayed_imports(serve_root)
rv = []
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
test_loader = testloader.TestLoader(test_paths, test_types,
testloader.TestFilter(), run_info)
for test_type, tests in test_loader.disabled_tests.iteritems():
for test in tests:
rv.append({"test": test.id, "reason": test.disabled()})
print json.dumps(rv, indent=2)
def list_disabled(tests_root, metadata_root, test_types, product, **kwargs):
do_test_relative_imports(tests_root)
rv = []
run_info = wpttest.get_run_info(metadata_root, product, debug=False)
test_loader = TestLoader(tests_root, metadata_root, test_types,
TestFilter(), run_info)
for test_type, tests in test_loader.disabled_tests.iteritems():
for test in tests:
rv.append({"test": test.id, "reason": test.disabled()})
print json.dumps(rv, indent=2)
def list_test_groups(tests_root, metadata_root, test_types, product, **kwargs):
do_test_relative_imports(tests_root)
run_info = wpttest.get_run_info(metadata_root, product, debug=False)
test_filter = TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = TestLoader(tests_root, metadata_root, test_types,
test_filter, run_info)
for item in sorted(test_loader.groups()):
print item
def list_disabled(serve_root, test_paths, test_types, product, **kwargs):
do_delayed_imports(serve_root)
rv = []
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
test_loader = testloader.TestLoader(test_paths,
test_types,
testloader.TestFilter(),
run_info)
for test_type, tests in test_loader.disabled_tests.iteritems():
for test in tests:
rv.append({"test": test.id, "reason": test.disabled()})
print json.dumps(rv, indent=2)
def list_test_groups(serve_root, test_paths, test_types, product, **kwargs):
do_delayed_imports(serve_root)
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
test_filter = testloader.TestFilter(
include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = testloader.TestLoader(test_paths, test_types, test_filter,
run_info)
for item in sorted(test_loader.groups(test_types)):
print item
def __init__(self, target, **kwargs):
self.target = target
self.test_type = kwargs["test_types"][0]
run_info = wpttest.get_run_info(kwargs["product"], debug=False)
test_filter = wptrunner.TestFilter(include=kwargs["include"])
self.test_loader = wptrunner.TestLoader(kwargs["tests_root"],
kwargs["metadata_root"],
test_filter,
run_info)
if kwargs["repeat"] == 1:
logger.critical("Need to specify --repeat with more than one repetition")
sys.exit(1)
self.kwargs = kwargs
def list_test_groups(serve_root, test_paths, test_types, product, **kwargs):
do_delayed_imports(serve_root)
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
test_filter = testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = testloader.TestLoader(test_paths,
test_types,
test_filter,
run_info)
for item in sorted(test_loader.groups()):
print item
def __init__(self, target, **kwargs):
self.target = target
self.test_type = kwargs["test_types"][0]
run_info = wpttest.get_run_info(kwargs["metadata_root"],
kwargs["product"],
debug=False)
test_filter = wptrunner.TestFilter(include=kwargs["include"])
self.test_loader = wptrunner.TestLoader(kwargs["tests_root"],
kwargs["metadata_root"],
test_filter, run_info)
if kwargs["repeat"] == 1:
logger.critical(
"Need to specify --repeat with more than one repetition")
sys.exit(1)
self.kwargs = kwargs
def get_loader(test_paths,
product,
debug=None,
run_info_extras=None,
**kwargs):
if run_info_extras is None:
run_info_extras = {}
run_info = wpttest.get_run_info(
kwargs["run_info"],
product,
browser_version=kwargs.get("browser_version"),
browser_channel=kwargs.get("browser_channel"),
verify=kwargs.get("verify"),
debug=debug,
extras=run_info_extras,
enable_webrender=kwargs.get("enable_webrender"))
test_manifests = testloader.ManifestLoader(
test_paths,
force_manifest_update=kwargs["manifest_update"],
manifest_download=kwargs["manifest_download"]).load()
manifest_filters = []
if kwargs["include"] or kwargs["exclude"] or kwargs[
"include_manifest"] or kwargs["default_exclude"]:
manifest_filters.append(
testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests,
explicit=kwargs["default_exclude"]))
ssl_enabled = sslutils.get_cls(kwargs["ssl_type"]).ssl_enabled
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
run_info,
manifest_filters=manifest_filters,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_enabled,
skip_timeout=kwargs["skip_timeout"])
return run_info, test_loader
def get_loader(test_paths, product, debug=False, **kwargs):
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"]).load()
test_filter = testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests)
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
test_filter,
run_info,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"])
return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=False, **kwargs):
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
test_manifests = testloader.ManifestLoader(
test_paths, force_manifest_update=kwargs["manifest_update"]).load()
test_filter = testloader.TestFilter(
include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests)
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
test_filter,
run_info,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_env.ssl_enabled)
return run_info, test_loader
def run_tests(config, test_paths, product, **kwargs):
with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
env.do_delayed_imports(logger, test_paths)
(check_args, target_browser_cls, get_browser_kwargs, executor_classes,
get_executor_kwargs, env_options, get_env_extras,
run_info_extras) = products.load_product(config, product)
ssl_env = env.ssl_env(logger, **kwargs)
env_extras = get_env_extras(**kwargs)
check_args(**kwargs)
if kwargs["install_fonts"]:
env_extras.append(
FontInstaller(font_dir=kwargs["font_dir"],
ahem=os.path.join(kwargs["tests_root"],
"fonts/Ahem.ttf")))
if "test_loader" in kwargs:
run_info = wpttest.get_run_info(
kwargs["run_info"],
product,
browser_version=kwargs.get("browser_version"),
debug=None,
extras=run_info_extras(**kwargs))
test_loader = kwargs["test_loader"]
else:
run_info, test_loader = get_loader(
test_paths,
product,
ssl_env,
run_info_extras=run_info_extras(**kwargs),
**kwargs)
test_source_kwargs = {"processes": kwargs["processes"]}
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs["depth"] = kwargs["run_by_dir"]
logger.info("Using %i client processes" % kwargs["processes"])
test_total = 0
unexpected_total = 0
kwargs["pause_after_test"] = get_pause_after_test(
test_loader, **kwargs)
with env.TestEnvironment(test_paths, ssl_env,
kwargs["pause_after_test"],
kwargs["debug_info"], env_options,
env_extras) as test_environment:
try:
test_environment.ensure_started()
except env.TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" %
e.message)
raise
repeat = kwargs["repeat"]
repeat_count = 0
repeat_until_unexpected = kwargs["repeat_until_unexpected"]
while repeat_count < repeat or repeat_until_unexpected:
repeat_count += 1
if repeat_until_unexpected:
logger.info("Repetition %i" % (repeat_count))
elif repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count, repeat))
test_count = 0
unexpected_count = 0
logger.suite_start(test_loader.test_ids,
name='web-platform-test',
run_info=run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
# WebDriver tests may create and destroy multiple browser
# processes as part of their expected behavior. These
# processes are managed by a WebDriver server binary. This
# obviates the need for wptrunner to provide a browser, so
# the NullBrowser is used in place of the "target" browser
if test_type == "wdspec":
browser_cls = NullBrowser
else:
browser_cls = target_browser_cls
browser_kwargs = get_browser_kwargs(
test_type,
run_info,
ssl_env=ssl_env,
config=test_environment.config,
**kwargs)
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(
test_type, test_environment.config,
test_environment.cache_manager, run_info, **kwargs)
if executor_cls is None:
logger.error(
"Unsupported test type %s for product %s" %
(test_type, product))
continue
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
if test_type == "testharness":
run_tests = {"testharness": []}
for test in test_loader.tests["testharness"]:
if test.testdriver and not executor_cls.supports_testdriver:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
elif test.jsshell and not executor_cls.supports_jsshell:
# We expect that tests for JavaScript shells
# will not be run along with tests that run in
# a full web browser, so we silently skip them
# here.
pass
else:
run_tests["testharness"].append(test)
else:
run_tests = test_loader.tests
with ManagerGroup("web-platform-tests",
kwargs["processes"], test_source_cls,
test_source_kwargs, browser_cls,
browser_kwargs, executor_cls,
executor_kwargs, kwargs["rerun"],
kwargs["pause_after_test"],
kwargs["pause_on_unexpected"],
kwargs["restart_on_unexpected"],
kwargs["debug_info"]) as manager_group:
try:
manager_group.run(test_type, run_tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
test_count += manager_group.test_count()
unexpected_count += manager_group.unexpected_count()
test_total += test_count
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
if repeat_until_unexpected and unexpected_total > 0:
break
logger.suite_end()
if test_total == 0:
logger.error("No tests ran")
return False
if unexpected_total and not kwargs["fail_on_unexpected"]:
logger.info("Tolerating %s unexpected results" % unexpected_total)
return True
return unexpected_total == 0
def run_tests(config, test_paths, product, **kwargs):
with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
env.do_delayed_imports(logger, test_paths)
(check_args,
target_browser_cls, get_browser_kwargs,
executor_classes, get_executor_kwargs,
env_options, get_env_extras, run_info_extras) = products.load_product(config, product)
ssl_env = env.ssl_env(logger, **kwargs)
env_extras = get_env_extras(**kwargs)
check_args(**kwargs)
if kwargs["install_fonts"]:
env_extras.append(FontInstaller(
font_dir=kwargs["font_dir"],
ahem=os.path.join(kwargs["tests_root"], "fonts/Ahem.ttf")
))
if "test_loader" in kwargs:
run_info = wpttest.get_run_info(kwargs["run_info"], product,
browser_version=kwargs.get("browser_version"),
debug=None,
extras=run_info_extras(**kwargs))
test_loader = kwargs["test_loader"]
else:
run_info, test_loader = get_loader(test_paths,
product,
ssl_env,
run_info_extras=run_info_extras(**kwargs),
**kwargs)
test_source_kwargs = {"processes": kwargs["processes"]}
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs["depth"] = kwargs["run_by_dir"]
logger.info("Using %i client processes" % kwargs["processes"])
test_total = 0
unexpected_total = 0
kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs)
with env.TestEnvironment(test_paths,
ssl_env,
kwargs["pause_after_test"],
kwargs["debug_info"],
env_options,
env_extras) as test_environment:
try:
test_environment.ensure_started()
except env.TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" % e.message)
raise
repeat = kwargs["repeat"]
repeat_count = 0
repeat_until_unexpected = kwargs["repeat_until_unexpected"]
while repeat_count < repeat or repeat_until_unexpected:
repeat_count += 1
if repeat_until_unexpected:
logger.info("Repetition %i" % (repeat_count))
elif repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count, repeat))
test_count = 0
unexpected_count = 0
logger.suite_start(test_loader.test_ids, name='web-platform-test', run_info=run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
# WebDriver tests may create and destroy multiple browser
# processes as part of their expected behavior. These
# processes are managed by a WebDriver server binary. This
# obviates the need for wptrunner to provide a browser, so
# the NullBrowser is used in place of the "target" browser
if test_type == "wdspec":
browser_cls = NullBrowser
else:
browser_cls = target_browser_cls
browser_kwargs = get_browser_kwargs(test_type,
run_info,
ssl_env=ssl_env,
config=test_environment.config,
**kwargs)
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(test_type,
test_environment.config,
test_environment.cache_manager,
run_info,
**kwargs)
if executor_cls is None:
logger.error("Unsupported test type %s for product %s" %
(test_type, product))
continue
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
if test_type == "testharness":
run_tests = {"testharness": []}
for test in test_loader.tests["testharness"]:
if test.testdriver and not executor_cls.supports_testdriver:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
elif test.jsshell and not executor_cls.supports_jsshell:
# We expect that tests for JavaScript shells
# will not be run along with tests that run in
# a full web browser, so we silently skip them
# here.
pass
else:
run_tests["testharness"].append(test)
else:
run_tests = test_loader.tests
with ManagerGroup("web-platform-tests",
kwargs["processes"],
test_source_cls,
test_source_kwargs,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
kwargs["rerun"],
kwargs["pause_after_test"],
kwargs["pause_on_unexpected"],
kwargs["restart_on_unexpected"],
kwargs["debug_info"]) as manager_group:
try:
manager_group.run(test_type, run_tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
test_count += manager_group.test_count()
unexpected_count += manager_group.unexpected_count()
test_total += test_count
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
if repeat_until_unexpected and unexpected_total > 0:
break
logger.suite_end()
if test_total == 0:
logger.error("No tests ran")
return False
if unexpected_total and not kwargs["fail_on_unexpected"]:
logger.info("Tolerating %s unexpected results" % unexpected_total)
return True
return unexpected_total == 0
def run_tests(config, tests_root, metadata_root, product, **kwargs):
logging_queue = None
logging_thread = None
original_stdio = (sys.stdout, sys.stderr)
test_queues = None
try:
if not kwargs["no_capture_stdio"]:
logging_queue = Queue()
logging_thread = LogThread(logging_queue, logger, "info")
sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT")
sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR")
logging_thread.start()
do_test_relative_imports(tests_root)
run_info = wpttest.get_run_info(metadata_root, product, debug=False)
(check_args,
browser_cls, get_browser_kwargs,
executor_classes, get_executor_kwargs,
env_options) = products.load_product(config, product)
check_args(**kwargs)
browser_kwargs = get_browser_kwargs(**kwargs)
unexpected_total = 0
if "test_loader" in kwargs:
test_loader = kwargs["test_loader"]
else:
test_filter = TestFilter(include=kwargs["include"], exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = TestLoader(tests_root, metadata_root, test_filter, run_info)
logger.info("Using %i client processes" % kwargs["processes"])
with TestEnvironment(tests_root, env_options) as test_environment:
try:
test_environment.ensure_started()
except TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" % e.message)
raise
base_server = "http://%s:%i" % (test_environment.config["host"],
test_environment.config["ports"]["http"][0])
repeat = kwargs["repeat"]
for repeat_count in xrange(repeat):
if repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
test_ids, test_queues = test_loader.queue_tests(kwargs["test_types"],
kwargs["chunk_type"],
kwargs["total_chunks"],
kwargs["this_chunk"])
unexpected_count = 0
logger.suite_start(test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
tests_queue = test_queues[test_type]
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(base_server,
**kwargs)
if executor_cls is None:
logger.error("Unsupported test type %s for product %s" %
(test_type, product))
continue
with ManagerGroup("web-platform-tests",
kwargs["processes"],
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
kwargs["pause_on_unexpected"]) as manager_group:
try:
manager_group.start(tests_queue)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
manager_group.wait()
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
logger.suite_end()
except KeyboardInterrupt:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.cancel_join_thread()
finally:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.close()
sys.stdout, sys.stderr = original_stdio
if not kwargs["no_capture_stdio"] and logging_queue is not None:
logger.info("Closing logging queue")
logging_queue.put(None)
if logging_thread is not None:
logging_thread.join(10)
logging_queue.close()
return manager_group.unexpected_count() == 0
def run_tests(config, test_paths, product, **kwargs):
with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
env.do_delayed_imports(logger, test_paths)
(check_args,
browser_cls, get_browser_kwargs,
executor_classes, get_executor_kwargs,
env_options) = products.load_product(config, product)
ssl_env = env.ssl_env(logger, **kwargs)
check_args(**kwargs)
if "test_loader" in kwargs:
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
test_loader = kwargs["test_loader"]
else:
run_info, test_loader = get_loader(test_paths, product, ssl_env,
**kwargs)
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
test_source_kwargs = {}
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs = {"depth": kwargs["run_by_dir"]}
logger.info("Using %i client processes" % kwargs["processes"])
unexpected_total = 0
kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs)
with env.TestEnvironment(test_paths,
ssl_env,
kwargs["pause_after_test"],
env_options) as test_environment:
try:
test_environment.ensure_started()
except env.TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" % e.message)
raise
browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
repeat = kwargs["repeat"]
for repeat_count in xrange(repeat):
if repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(test_type,
test_environment.external_config,
test_environment.cache_manager,
**kwargs)
if executor_cls is None:
logger.error("Unsupported test type %s for product %s" %
(test_type, product))
continue
with ManagerGroup("web-platform-tests",
kwargs["processes"],
test_source_cls,
test_source_kwargs,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
kwargs["pause_after_test"],
kwargs["pause_on_unexpected"],
kwargs["debug_args"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
logger.suite_end()
return unexpected_total == 0
def run_tests(config, test_paths, product, **kwargs):
with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
env.do_delayed_imports(logger, test_paths)
(check_args,
browser_cls, get_browser_kwargs,
executor_classes, get_executor_kwargs,
env_options, run_info_extras) = products.load_product(config, product)
ssl_env = env.ssl_env(logger, **kwargs)
check_args(**kwargs)
if "test_loader" in kwargs:
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=None,
extras=run_info_extras(**kwargs))
test_loader = kwargs["test_loader"]
else:
run_info, test_loader = get_loader(test_paths,
product,
ssl_env,
run_info_extras=run_info_extras(**kwargs),
**kwargs)
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
test_source_kwargs = {}
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs = {"depth": kwargs["run_by_dir"]}
logger.info("Using %i client processes" % kwargs["processes"])
unexpected_total = 0
kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs)
with env.TestEnvironment(test_paths,
ssl_env,
kwargs["pause_after_test"],
kwargs["debug_info"],
env_options) as test_environment:
try:
test_environment.ensure_started()
except env.TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" % e.message)
raise
browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
repeat = kwargs["repeat"]
for repeat_count in xrange(repeat):
if repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(test_type,
test_environment.external_config,
test_environment.cache_manager,
run_info,
**kwargs)
if executor_cls is None:
logger.error("Unsupported test type %s for product %s" %
(test_type, product))
continue
with ManagerGroup("web-platform-tests",
kwargs["processes"],
test_source_cls,
test_source_kwargs,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
kwargs["pause_after_test"],
kwargs["pause_on_unexpected"],
kwargs["debug_info"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
logger.suite_end()
return unexpected_total == 0
def run_tests(config, test_paths, product, **kwargs):
with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
env.do_delayed_imports(logger, test_paths)
(check_args, target_browser_cls, get_browser_kwargs, executor_classes,
get_executor_kwargs, env_options,
run_info_extras) = products.load_product(config, product)
ssl_env = env.ssl_env(logger, **kwargs)
check_args(**kwargs)
if "test_loader" in kwargs:
run_info = wpttest.get_run_info(kwargs["run_info"],
product,
debug=None,
extras=run_info_extras(**kwargs))
test_loader = kwargs["test_loader"]
else:
run_info, test_loader = get_loader(
test_paths,
product,
ssl_env,
run_info_extras=run_info_extras(**kwargs),
**kwargs)
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
test_source_kwargs = {}
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs = {"depth": kwargs["run_by_dir"]}
logger.info("Using %i client processes" % kwargs["processes"])
unexpected_total = 0
kwargs["pause_after_test"] = get_pause_after_test(
test_loader, **kwargs)
with env.TestEnvironment(test_paths, ssl_env,
kwargs["pause_after_test"],
kwargs["debug_info"],
env_options) as test_environment:
try:
test_environment.ensure_started()
except env.TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" %
e.message)
raise
repeat = kwargs["repeat"]
repeat_count = 0
repeat_until_unexpected = kwargs["repeat_until_unexpected"]
while repeat_count < repeat or repeat_until_unexpected:
repeat_count += 1
if repeat_until_unexpected:
logger.info("Repetition %i" % (repeat_count))
elif repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
# WebDriver tests may create and destroy multiple browser
# processes as part of their expected behavior. These
# processes are managed by a WebDriver server binary. This
# obviates the need for wptrunner to provide a browser, so
# the NullBrowser is used in place of the "target" browser
if test_type == "wdspec":
browser_cls = NullBrowser
else:
browser_cls = target_browser_cls
browser_kwargs = get_browser_kwargs(test_type,
run_info,
ssl_env=ssl_env,
**kwargs)
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(
test_type, test_environment.external_config,
test_environment.cache_manager, run_info, **kwargs)
if executor_cls is None:
logger.error(
"Unsupported test type %s for product %s" %
(test_type, product))
continue
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
with ManagerGroup("web-platform-tests",
kwargs["processes"], test_source_cls,
test_source_kwargs, browser_cls,
browser_kwargs, executor_cls,
executor_kwargs,
kwargs["pause_after_test"],
kwargs["pause_on_unexpected"],
kwargs["restart_on_unexpected"],
kwargs["debug_info"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
if repeat_until_unexpected and unexpected_total > 0:
break
logger.suite_end()
return unexpected_total == 0
def run_tests(config, serve_root, test_paths, product, **kwargs):
logging_queue = None
logging_thread = None
original_stdio = (sys.stdout, sys.stderr)
test_queues = None
try:
if not kwargs["no_capture_stdio"]:
logging_queue = Queue()
logging_thread = LogThread(logging_queue, logger, "info")
sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT")
sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR")
logging_thread.start()
do_delayed_imports(serve_root)
run_info = wpttest.get_run_info(kwargs["run_info"],
product,
debug=False)
(check_args, browser_cls, get_browser_kwargs, executor_classes,
get_executor_kwargs,
env_options) = products.load_product(config, product)
check_args(**kwargs)
ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
unexpected_total = 0
if "test_loader" in kwargs:
test_loader = kwargs["test_loader"]
else:
test_filter = testloader.TestFilter(
include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = testloader.TestLoader(
test_paths, kwargs["test_types"], test_filter, run_info,
kwargs["chunk_type"], kwargs["total_chunks"],
kwargs["this_chunk"], kwargs["manifest_update"])
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
test_source_kwargs = {}
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs = {"depth": kwargs["run_by_dir"]}
logger.info("Using %i client processes" % kwargs["processes"])
with TestEnvironment(serve_root, test_paths, ssl_env,
env_options) as test_environment:
try:
test_environment.ensure_started()
except TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" %
e.message)
raise
browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
base_server = "http://%s:%i" % (
test_environment.external_config["host"],
test_environment.external_config["ports"]["http"][0])
repeat = kwargs["repeat"]
for repeat_count in xrange(repeat):
if repeat > 1:
logger.info("Repetition %i / %i" %
(repeat_count + 1, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(
base_server, **kwargs)
if executor_cls is None:
logger.error(
"Unsupported test type %s for product %s" %
(test_type, product))
continue
with ManagerGroup(
"web-platform-tests", kwargs["processes"],
test_source_cls, test_source_kwargs, browser_cls,
browser_kwargs, executor_cls, executor_kwargs,
kwargs["pause_on_unexpected"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
logger.suite_end()
except KeyboardInterrupt:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.cancel_join_thread()
finally:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.close()
sys.stdout, sys.stderr = original_stdio
if not kwargs["no_capture_stdio"] and logging_queue is not None:
logger.info("Closing logging queue")
logging_queue.put(None)
if logging_thread is not None:
logging_thread.join(10)
logging_queue.close()
return manager_group.unexpected_count() == 0
def run_tests(config, test_paths, product, **kwargs):
with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
env.do_delayed_imports(logger, test_paths)
(check_args,
target_browser_cls, get_browser_kwargs,
executor_classes, get_executor_kwargs,
env_options, get_env_extras, run_info_extras) = products.load_product(config, product)
ssl_env = env.ssl_env(logger, **kwargs)
env_extras = get_env_extras(**kwargs)
check_args(**kwargs)
if "test_loader" in kwargs:
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=None,
extras=run_info_extras(**kwargs))
test_loader = kwargs["test_loader"]
else:
run_info, test_loader = get_loader(test_paths,
product,
ssl_env,
run_info_extras=run_info_extras(**kwargs),
**kwargs)
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
test_source_kwargs = {}
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs = {"depth": kwargs["run_by_dir"]}
logger.info("Using %i client processes" % kwargs["processes"])
unexpected_total = 0
kwargs["pause_after_test"] = get_pause_after_test(test_loader, **kwargs)
with env.TestEnvironment(test_paths,
ssl_env,
kwargs["pause_after_test"],
kwargs["debug_info"],
env_options,
env_extras) as test_environment:
try:
test_environment.ensure_started()
except env.TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" % e.message)
raise
browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
repeat = kwargs["repeat"]
repeat_count = 0
repeat_until_unexpected = kwargs["repeat_until_unexpected"]
while repeat_count < repeat or repeat_until_unexpected:
repeat_count += 1
if repeat_until_unexpected:
logger.info("Repetition %i" % (repeat_count))
elif repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
# WebDriver tests may create and destroy multiple browser
# processes as part of their expected behavior. These
# processes are managed by a WebDriver server binary. This
# obviates the need for wptrunner to provide a browser, so
# the NullBrowser is used in place of the "target" browser
if test_type == "wdspec":
browser_cls = NullBrowser
else:
browser_cls = target_browser_cls
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(test_type,
test_environment.external_config,
test_environment.cache_manager,
run_info,
**kwargs)
if executor_cls is None:
logger.error("Unsupported test type %s for product %s" %
(test_type, product))
continue
with ManagerGroup("web-platform-tests",
kwargs["processes"],
test_source_cls,
test_source_kwargs,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
kwargs["pause_after_test"],
kwargs["pause_on_unexpected"],
kwargs["restart_on_unexpected"],
kwargs["debug_info"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
if repeat_until_unexpected and unexpected_total > 0:
break
logger.suite_end()
return unexpected_total == 0
def run_tests(config, serve_root, test_paths, product, **kwargs):
logging_queue = None
logging_thread = None
original_stdio = (sys.stdout, sys.stderr)
test_queues = None
try:
if not kwargs["no_capture_stdio"]:
logging_queue = Queue()
logging_thread = LogThread(logging_queue, logger, "info")
sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT")
sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR")
logging_thread.start()
do_delayed_imports(serve_root)
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
(check_args,
browser_cls, get_browser_kwargs,
executor_classes, get_executor_kwargs,
env_options) = products.load_product(config, product)
check_args(**kwargs)
ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
unexpected_total = 0
if "test_loader" in kwargs:
test_loader = kwargs["test_loader"]
else:
test_filter = testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = testloader.TestLoader(test_paths,
kwargs["test_types"],
test_filter,
run_info,
kwargs["chunk_type"],
kwargs["total_chunks"],
kwargs["this_chunk"],
kwargs["manifest_update"])
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
test_source_kwargs = {}
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs = {"depth": kwargs["run_by_dir"]}
logger.info("Using %i client processes" % kwargs["processes"])
with TestEnvironment(serve_root,
test_paths,
ssl_env,
env_options) as test_environment:
try:
test_environment.ensure_started()
except TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" % e.message)
raise
browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
base_server = "http://%s:%i" % (test_environment.external_config["host"],
test_environment.external_config["ports"]["http"][0])
repeat = kwargs["repeat"]
for repeat_count in xrange(repeat):
if repeat > 1:
logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(base_server,
**kwargs)
if executor_cls is None:
logger.error("Unsupported test type %s for product %s" %
(test_type, product))
continue
with ManagerGroup("web-platform-tests",
kwargs["processes"],
test_source_cls,
test_source_kwargs,
browser_cls,
browser_kwargs,
executor_cls,
executor_kwargs,
kwargs["pause_on_unexpected"],
kwargs["debug_args"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
logger.suite_end()
except KeyboardInterrupt:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.cancel_join_thread()
finally:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.close()
sys.stdout, sys.stderr = original_stdio
if not kwargs["no_capture_stdio"] and logging_queue is not None:
logger.info("Closing logging queue")
logging_queue.put(None)
if logging_thread is not None:
logging_thread.join(10)
logging_queue.close()
return unexpected_total == 0
