def get_loader(test_paths, product, ssl_env, debug=None, **kwargs):
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
test_manifests = testloader.ManifestLoader(
test_paths, force_manifest_update=kwargs["manifest_update"]).load()
manifest_filters = []
meta_filters = []
if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
manifest_filters.append(
testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests))
if kwargs["tags"]:
meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
run_info,
manifest_filters=manifest_filters,
meta_filters=meta_filters,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_env.ssl_enabled)
return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=None, run_info_extras=None, **kwargs):
if run_info_extras is None:
run_info_extras = {}
run_info = wpttest.get_run_info(kwargs["run_info"], product,
browser_version=kwargs.get("browser_version"),
debug=debug,
extras=run_info_extras)
test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"],
manifest_download=kwargs["manifest_download"]).load()
manifest_filters = []
meta_filters = []
if kwargs["include"] or kwargs["exclude"] or kwargs["include_manifest"]:
manifest_filters.append(testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests))
if kwargs["tags"]:
meta_filters.append(testloader.TagFilter(tags=kwargs["tags"]))
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
run_info,
manifest_filters=manifest_filters,
meta_filters=meta_filters,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_env.ssl_enabled,
skip_timeout=kwargs["skip_timeout"])
return run_info, test_loader
def list_disabled(serve_root, test_paths, test_types, product, **kwargs):
do_delayed_imports(serve_root)
rv = []
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
test_loader = testloader.TestLoader(test_paths, test_types,
testloader.TestFilter(), run_info)
for test_type, tests in test_loader.disabled_tests.iteritems():
for test in tests:
rv.append({"test": test.id, "reason": test.disabled()})
print json.dumps(rv, indent=2)
def list_test_groups(serve_root, test_paths, test_types, product, **kwargs):
do_delayed_imports(serve_root)
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
test_filter = testloader.TestFilter(
include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = testloader.TestLoader(test_paths, test_types, test_filter,
run_info)
for item in sorted(test_loader.groups(test_types)):
print item
def get_loader(test_paths,
product,
debug=None,
run_info_extras=None,
**kwargs):
if run_info_extras is None:
run_info_extras = {}
run_info = wpttest.get_run_info(
kwargs["run_info"],
product,
browser_version=kwargs.get("browser_version"),
browser_channel=kwargs.get("browser_channel"),
verify=kwargs.get("verify"),
debug=debug,
extras=run_info_extras,
enable_webrender=kwargs.get("enable_webrender"))
test_manifests = testloader.ManifestLoader(
test_paths,
force_manifest_update=kwargs["manifest_update"],
manifest_download=kwargs["manifest_download"]).load()
manifest_filters = []
if kwargs["include"] or kwargs["exclude"] or kwargs[
"include_manifest"] or kwargs["default_exclude"]:
manifest_filters.append(
testloader.TestFilter(include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests,
explicit=kwargs["default_exclude"]))
ssl_enabled = sslutils.get_cls(kwargs["ssl_type"]).ssl_enabled
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
run_info,
manifest_filters=manifest_filters,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_enabled,
skip_timeout=kwargs["skip_timeout"])
return run_info, test_loader
def get_loader(test_paths, product, ssl_env, debug=False, **kwargs):
run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
test_manifests = testloader.ManifestLoader(
test_paths, force_manifest_update=kwargs["manifest_update"]).load()
test_filter = testloader.TestFilter(
include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"],
test_manifests=test_manifests)
test_loader = testloader.TestLoader(test_manifests,
kwargs["test_types"],
test_filter,
run_info,
chunk_type=kwargs["chunk_type"],
total_chunks=kwargs["total_chunks"],
chunk_number=kwargs["this_chunk"],
include_https=ssl_env.ssl_enabled)
return run_info, test_loader
def run_tests(config, serve_root, test_paths, product, **kwargs):
logging_queue = None
logging_thread = None
original_stdio = (sys.stdout, sys.stderr)
test_queues = None
try:
if not kwargs["no_capture_stdio"]:
logging_queue = Queue()
logging_thread = LogThread(logging_queue, logger, "info")
sys.stdout = LoggingWrapper(logging_queue, prefix="STDOUT")
sys.stderr = LoggingWrapper(logging_queue, prefix="STDERR")
logging_thread.start()
do_delayed_imports(serve_root)
run_info = wpttest.get_run_info(kwargs["run_info"],
product,
debug=False)
(check_args, browser_cls, get_browser_kwargs, executor_classes,
get_executor_kwargs,
env_options) = products.load_product(config, product)
check_args(**kwargs)
ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
unexpected_total = 0
if "test_loader" in kwargs:
test_loader = kwargs["test_loader"]
else:
test_filter = testloader.TestFilter(
include=kwargs["include"],
exclude=kwargs["exclude"],
manifest_path=kwargs["include_manifest"])
test_loader = testloader.TestLoader(
test_paths, kwargs["test_types"], test_filter, run_info,
kwargs["chunk_type"], kwargs["total_chunks"],
kwargs["this_chunk"], kwargs["manifest_update"])
if kwargs["run_by_dir"] is False:
test_source_cls = testloader.SingleTestSource
test_source_kwargs = {}
else:
# A value of None indicates infinite depth
test_source_cls = testloader.PathGroupedSource
test_source_kwargs = {"depth": kwargs["run_by_dir"]}
logger.info("Using %i client processes" % kwargs["processes"])
with TestEnvironment(serve_root, test_paths, ssl_env,
env_options) as test_environment:
try:
test_environment.ensure_started()
except TestEnvironmentError as e:
logger.critical("Error starting test environment: %s" %
e.message)
raise
browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
base_server = "http://%s:%i" % (
test_environment.external_config["host"],
test_environment.external_config["ports"]["http"][0])
repeat = kwargs["repeat"]
for repeat_count in xrange(repeat):
if repeat > 1:
logger.info("Repetition %i / %i" %
(repeat_count + 1, repeat))
unexpected_count = 0
logger.suite_start(test_loader.test_ids, run_info)
for test_type in kwargs["test_types"]:
logger.info("Running %s tests" % test_type)
for test in test_loader.disabled_tests[test_type]:
logger.test_start(test.id)
logger.test_end(test.id, status="SKIP")
executor_cls = executor_classes.get(test_type)
executor_kwargs = get_executor_kwargs(
base_server, **kwargs)
if executor_cls is None:
logger.error(
"Unsupported test type %s for product %s" %
(test_type, product))
continue
with ManagerGroup(
"web-platform-tests", kwargs["processes"],
test_source_cls, test_source_kwargs, browser_cls,
browser_kwargs, executor_cls, executor_kwargs,
kwargs["pause_on_unexpected"]) as manager_group:
try:
manager_group.run(test_type, test_loader.tests)
except KeyboardInterrupt:
logger.critical("Main thread got signal")
manager_group.stop()
raise
unexpected_count += manager_group.unexpected_count()
unexpected_total += unexpected_count
logger.info("Got %i unexpected results" % unexpected_count)
logger.suite_end()
except KeyboardInterrupt:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.cancel_join_thread()
finally:
if test_queues is not None:
for queue in test_queues.itervalues():
queue.close()
sys.stdout, sys.stderr = original_stdio
if not kwargs["no_capture_stdio"] and logging_queue is not None:
logger.info("Closing logging queue")
logging_queue.put(None)
if logging_thread is not None:
logging_thread.join(10)
logging_queue.close()
return manager_group.unexpected_count() == 0