def create_parser_update(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(
"web-platform-tests-update", description="Update script for web-platform-tests tests."
)
parser.add_argument(
"--product",
action="store",
choices=product_choices,
default=None,
help="Browser for which metadata is being updated",
)
parser.add_argument("--config", action="store", type=abs_path, help="Path to config file")
parser.add_argument(
"--metadata",
action="store",
type=abs_path,
dest="metadata_root",
help="Path to the folder containing test metadata",
),
parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to web-platform-tests"),
parser.add_argument(
"--sync-path",
action="store",
type=abs_path,
help="Path to store git checkout of web-platform-tests during update",
),
parser.add_argument("--remote_url", action="store", help="URL of web-platfrom-tests repository to sync against"),
parser.add_argument("--branch", action="store", type=abs_path, help="Remote branch to sync against")
parser.add_argument("--rev", action="store", help="Revision to sync to")
parser.add_argument(
"--no-patch", action="store_true", help="Don't create an mq patch or git commit containing the changes."
)
parser.add_argument(
"--sync", dest="sync", action="store_true", default=False, help="Sync the tests with the latest from upstream"
)
parser.add_argument(
"--ignore-existing",
action="store_true",
help="When updating test results only consider results from the logfiles provided, not existing expectations.",
)
parser.add_argument(
"--continue", action="store_true", help="Continue a previously started run of the update script"
)
parser.add_argument(
"--abort", action="store_true", help="Clear state from a previous incomplete run of the update script"
)
# Should make this required iff run=logfile
parser.add_argument("run_log", nargs="*", type=abs_path, help="Log file from run of tests")
commandline.add_logging_group(parser)
return parser
def create_parser_update(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser("web-platform-tests-update",
description="Update script for web-platform-tests tests.")
parser.add_argument("--product", action="store", choices=product_choices,
default=None, help="Browser for which metadata is being updated")
parser.add_argument("--config", action="store", type=abs_path, help="Path to config file")
parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
help="Path to the folder containing test metadata"),
parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
help="Path to web-platform-tests"),
parser.add_argument("--manifest", action="store", type=abs_path, dest="manifest_path",
help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)")
parser.add_argument("--sync-path", action="store", type=abs_path,
help="Path to store git checkout of web-platform-tests during update"),
parser.add_argument("--remote_url", action="store",
help="URL of web-platfrom-tests repository to sync against"),
parser.add_argument("--branch", action="store", type=abs_path,
help="Remote branch to sync against")
parser.add_argument("--rev", action="store", help="Revision to sync to")
parser.add_argument("--patch", action="store_true", dest="patch", default=None,
help="Create a VCS commit containing the changes.")
parser.add_argument("--no-patch", action="store_false", dest="patch",
help="Don't create a VCS commit containing the changes.")
parser.add_argument("--sync", dest="sync", action="store_true", default=False,
help="Sync the tests with the latest from upstream (implies --patch)")
parser.add_argument("--ignore-existing", action="store_true",
help="When updating test results only consider results from the logfiles provided, not existing expectations.")
parser.add_argument("--stability", nargs="?", action="store", const="unstable", default=None,
help=("Reason for disabling tests. When updating test results, disable tests that have "
"inconsistent results across many runs with the given reason."))
parser.add_argument("--no-remove-obsolete", action="store_false", dest="remove_obsolete", default=True,
help=("Don't remove metadata files that no longer correspond to a test file"))
parser.add_argument("--no-store-state", action="store_false", dest="store_state",
help="Store state so that steps can be resumed after failure")
parser.add_argument("--continue", action="store_true",
help="Continue a previously started run of the update script")
parser.add_argument("--abort", action="store_true",
help="Clear state from a previous incomplete run of the update script")
parser.add_argument("--exclude", action="store", nargs="*",
help="List of glob-style paths to exclude when syncing tests")
parser.add_argument("--include", action="store", nargs="*",
help="List of glob-style paths to include which would otherwise be excluded when syncing tests")
parser.add_argument("--extra-property", action="append", default=[],
help="Extra property from run_info.json to use in metadata update")
# Should make this required iff run=logfile
parser.add_argument("run_log", nargs="*", type=abs_path,
help="Log file from run of tests")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.")
parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
help="Path to the folder containing test metadata"),
parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
help="Path to test files"),
parser.add_argument("--run-info", action="store", type=abs_path,
help="Path to directory containing extra json files to add to run info")
parser.add_argument("--config", action="store", type=abs_path, dest="config",
help="Path to config file")
parser.add_argument("--manifest-update", action="store_true", default=False,
help="Force regeneration of the test manifest")
parser.add_argument("--binary", action="store",
type=abs_path, help="Binary to run tests against")
parser.add_argument("--webdriver-binary", action="store", metavar="BINARY",
type=abs_path, help="WebDriver server binary to use")
parser.add_argument("--processes", action="store", type=int, default=None,
help="Number of simultaneous processes to use")
parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument("--repeat", action="store", type=int, default=1,
help="Number of times to run the tests")
parser.add_argument("--no-capture-stdio", action="store_true", default=False,
help="Don't capture stdio and write to logging")
parser.add_argument("--product", action="store", choices=product_choices,
default=None, help="Browser against which to run tests")
parser.add_argument("--list-test-groups", action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
parser.add_argument("--list-disabled", action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
build_type = parser.add_mutually_exclusive_group()
build_type.add_argument("--debug-build", dest="debug", action="store_true",
default=None,
help="Build is a debug build (overrides any mozinfo file)")
build_type.add_argument("--release-build", dest="debug", action="store_false",
default=None,
help="Build is a release (overrides any mozinfo file)")
test_selection_group = parser.add_argument_group("Test Selection")
test_selection_group.add_argument("--test-types", action="store",
nargs="*", default=["testharness", "reftest"],
choices=["testharness", "reftest"],
help="Test types to run")
test_selection_group.add_argument("--include", action="append",
help="URL prefix to include")
test_selection_group.add_argument("--exclude", action="append",
help="URL prefix to exclude")
test_selection_group.add_argument("--include-manifest", type=abs_path,
help="Path to manifest listing tests to include")
test_selection_group.add_argument("--tag", action="append", dest="tags",
help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument('--debugger', const="__default__", nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
help="Halt the test runner after each test (this happens by default if only a single test is run)")
debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
help="Don't halt the test runner irrespective of the number of tests run")
debugging_group.add_argument('--pause-on-unexpected', action="store_true",
help="Halt the test runner when an unexpected result is encountered")
debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
chunking_group = parser.add_argument_group("Test Chunking")
chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
help="Total number of chunks to use")
chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
help="Chunk number to run")
chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"],
default=None, help="Chunking type to use")
ssl_group = parser.add_argument_group("SSL/TLS")
ssl_group.add_argument("--ssl-type", action="store", default=None,
choices=["openssl", "pregenerated", "none"],
help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
ssl_group.add_argument("--openssl-binary", action="store",
help="Path to openssl binary", default="openssl")
ssl_group.add_argument("--certutil-binary", action="store",
help="Path to certutil binary for use with Firefox + ssl")
ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path,
help="Path to ca certificate when using pregenerated ssl certificates")
ssl_group.add_argument("--host-key-path", action="store", type=abs_path,
help="Path to host private key when using pregenerated ssl certificates")
ssl_group.add_argument("--host-cert-path", action="store", type=abs_path,
help="Path to host certificate when using pregenerated ssl certificates")
gecko_group = parser.add_argument_group("Gecko-specific")
gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
help="Path to the folder containing browser prefs")
b2g_group = parser.add_argument_group("B2G-specific")
b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False,
help="Don't backup device before testrun with --product=b2g")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument("--user-stylesheet",
default=[], action="append", dest="user_stylesheets",
help="Inject a user CSS stylesheet into every test.")
parser.add_argument("test_list", nargs="*",
help="List of URLs for tests to run, or paths including tests to run. "
"(equivalent to --include)")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""",
usage="""%(prog)s [OPTION]... [TEST]...
TEST is either the full path to a test file to run, or the URL of a test excluding
scheme host and port.""")
parser.add_argument("--manifest-update", action="store_true", default=None,
help="Regenerate the test manifest.")
parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update",
help="Prevent regeneration of the test manifest.")
parser.add_argument("--manifest-download", action="store_true", default=None,
help="Attempt to download a preexisting manifest when updating.")
parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--processes", action="store", type=int, default=None,
help="Number of simultaneous processes to use")
parser.add_argument("--no-capture-stdio", action="store_true", default=False,
help="Don't capture stdio and write to logging")
parser.add_argument("--no-fail-on-unexpected", action="store_false",
default=True,
dest="fail_on_unexpected",
help="Exit with status code 0 when test expectations are violated")
mode_group = parser.add_argument_group("Mode")
mode_group.add_argument("--list-test-groups", action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
mode_group.add_argument("--list-disabled", action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
mode_group.add_argument("--list-tests", action="store_true",
default=False,
help="List all tests that will run")
mode_group.add_argument("--verify", action="store_true",
default=False,
help="Run a stability check on the selected tests")
mode_group.add_argument("--verify-log-full", action="store_true",
default=False,
help="Output per-iteration test results when running verify")
mode_group.add_argument("--verify-repeat-loop", action="store",
default=10,
help="Number of iterations for a run that reloads each test without restart.",
type=int)
mode_group.add_argument("--verify-repeat-restart", action="store",
default=5,
help="Number of iterations, for a run that restarts the runner between each iteration",
type=int)
chaos_mode_group = mode_group.add_mutually_exclusive_group()
chaos_mode_group.add_argument("--verify-no-chaos-mode", action="store_false",
default=True,
dest="verify_chaos_mode",
help="Disable chaos mode when running on Firefox")
chaos_mode_group.add_argument("--verify-chaos-mode", action="store_true",
default=True,
dest="verify_chaos_mode",
help="Enable chaos mode when running on Firefox")
mode_group.add_argument("--verify-max-time", action="store",
default=None,
help="The maximum number of minutes for the job to run",
type=lambda x: timedelta(minutes=float(x)))
output_results_group = mode_group.add_mutually_exclusive_group()
output_results_group.add_argument("--verify-no-output-results", action="store_false",
dest="verify_output_results",
default=True,
help="Prints individuals test results and messages")
output_results_group.add_argument("--verify-output-results", action="store_true",
dest="verify_output_results",
default=True,
help="Disable printing individuals test results and messages")
test_selection_group = parser.add_argument_group("Test Selection")
test_selection_group.add_argument("--test-types", action="store",
nargs="*", default=wpttest.enabled_tests,
choices=wpttest.enabled_tests,
help="Test types to run")
test_selection_group.add_argument("--include", action="append",
help="URL prefix to include")
test_selection_group.add_argument("--exclude", action="append",
help="URL prefix to exclude")
test_selection_group.add_argument("--include-manifest", type=abs_path,
help="Path to manifest listing tests to include")
test_selection_group.add_argument("--skip-timeout", action="store_true",
help="Skip tests that are expected to time out")
test_selection_group.add_argument("--tag", action="append", dest="tags",
help="Labels applied to tests to include in the run. "
"Labels starting dir: are equivalent to top-level directories.")
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument('--debugger', const="__default__", nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
debugging_group.add_argument("--rerun", action="store", type=int, default=1,
help="Number of times to re run each test without restarts")
debugging_group.add_argument("--repeat", action="store", type=int, default=1,
help="Number of times to run the tests, restarting between each run")
debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
help="Run tests in a loop until one returns an unexpected result")
debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
help="Halt the test runner after each test (this happens by default if only a single test is run)")
debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
help="Don't halt the test runner irrespective of the number of tests run")
debugging_group.add_argument('--pause-on-unexpected', action="store_true",
help="Halt the test runner when an unexpected result is encountered")
debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected",
default=True, action="store_false",
help="Don't restart on an unexpected result")
debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
debugging_group.add_argument("--pdb", action="store_true",
help="Drop into pdb on python exception")
config_group = parser.add_argument_group("Configuration")
config_group.add_argument("--binary", action="store",
type=abs_path, help="Binary to run tests against")
config_group.add_argument('--binary-arg',
default=[], action="append", dest="binary_args",
help="Extra argument for the binary")
config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY",
type=abs_path, help="WebDriver server binary to use")
config_group.add_argument('--webdriver-arg',
default=[], action="append", dest="webdriver_args",
help="Extra argument for the WebDriver binary")
config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
help="Path to root directory containing test metadata"),
config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
help="Path to root directory containing test files"),
config_group.add_argument("--manifest", action="store", type=abs_path, dest="manifest_path",
help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)")
config_group.add_argument("--run-info", action="store", type=abs_path,
help="Path to directory containing extra json files to add to run info")
config_group.add_argument("--product", action="store", choices=product_choices,
default=None, help="Browser against which to run tests")
config_group.add_argument("--config", action="store", type=abs_path, dest="config",
help="Path to config file")
config_group.add_argument("--install-fonts", action="store_true",
default=None,
help="Allow the wptrunner to install fonts on your system")
config_group.add_argument("--font-dir", action="store", type=abs_path, dest="font_dir",
help="Path to local font installation directory", default=None)
build_type = parser.add_mutually_exclusive_group()
build_type.add_argument("--debug-build", dest="debug", action="store_true",
default=None,
help="Build is a debug build (overrides any mozinfo file)")
build_type.add_argument("--release-build", dest="debug", action="store_false",
default=None,
help="Build is a release (overrides any mozinfo file)")
chunking_group = parser.add_argument_group("Test Chunking")
chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
help="Total number of chunks to use")
chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
help="Chunk number to run")
chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"],
default=None, help="Chunking type to use")
ssl_group = parser.add_argument_group("SSL/TLS")
ssl_group.add_argument("--ssl-type", action="store", default=None,
choices=["openssl", "pregenerated", "none"],
help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
ssl_group.add_argument("--openssl-binary", action="store",
help="Path to openssl binary", default="openssl")
ssl_group.add_argument("--certutil-binary", action="store",
help="Path to certutil binary for use with Firefox + ssl")
ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path,
help="Path to ca certificate when using pregenerated ssl certificates")
ssl_group.add_argument("--host-key-path", action="store", type=abs_path,
help="Path to host private key when using pregenerated ssl certificates")
ssl_group.add_argument("--host-cert-path", action="store", type=abs_path,
help="Path to host certificate when using pregenerated ssl certificates")
gecko_group = parser.add_argument_group("Gecko-specific")
gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
help="Path to the folder containing browser prefs")
gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
help="Run tests without electrolysis preferences")
gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store",
help="Path to directory containing assertion stack fixing scripts")
gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
default=[], metavar="PREF=VALUE",
help="Defines an extra user preference (overrides those in prefs_root)")
gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
help="Enable leak checking")
gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1,
help="Number of parallel threads to use for stylo")
gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true",
default=None, help="Enable reftest runner implemented inside Marionette")
gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false",
help="Disable reftest runner implemented inside Marionette")
gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store",
choices=["always", "fail", "unexpected"], default="unexpected",
help="With --reftest-internal, when to take a screenshot")
gecko_group.add_argument("--chaos", dest="chaos_mode_flags", action="store",
nargs="?", const=0xFFFFFFFF, type=int,
help="Enable chaos mode with the specified feature flag "
"(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for "
"details). If no value is supplied, all features are activated")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument("--user-stylesheet",
default=[], action="append", dest="user_stylesheets",
help="Inject a user CSS stylesheet into every test.")
sauce_group = parser.add_argument_group("Sauce Labs-specific")
sauce_group.add_argument("--sauce-browser", dest="sauce_browser",
help="Sauce Labs browser name")
sauce_group.add_argument("--sauce-platform", dest="sauce_platform",
help="Sauce Labs OS platform")
sauce_group.add_argument("--sauce-version", dest="sauce_version",
help="Sauce Labs browser version")
sauce_group.add_argument("--sauce-build", dest="sauce_build",
help="Sauce Labs build identifier")
sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*",
help="Sauce Labs identifying tag", default=[])
sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id",
help="Sauce Connect tunnel identifier")
sauce_group.add_argument("--sauce-user", dest="sauce_user",
help="Sauce Labs user name")
sauce_group.add_argument("--sauce-key", dest="sauce_key",
default=os.environ.get("SAUCE_ACCESS_KEY"),
help="Sauce Labs access key")
sauce_group.add_argument("--sauce-connect-binary",
dest="sauce_connect_binary",
help="Path to Sauce Connect binary")
webkit_group = parser.add_argument_group("WebKit-specific")
webkit_group.add_argument("--webkit-port", dest="webkit_port",
help="WebKit port")
parser.add_argument("test_list", nargs="*",
help="List of URLs for tests to run, or paths including tests to run. "
"(equivalent to --include)")
commandline.log_formatters["wptreport"] = (formatters.WptreportFormatter, "wptreport format")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.")
parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
help="Path to the folder containing test metadata"),
parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
help="Path to web-platform-tests"),
parser.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
help="Path to the folder containing browser prefs"),
parser.add_argument("--config", action="store", type=abs_path,
help="Path to config file")
parser.add_argument("--binary", action="store",
type=abs_path, help="Binary to run tests against")
parser.add_argument("--test-types", action="store",
nargs="*", default=["testharness", "reftest"],
choices=["testharness", "reftest"],
help="Test types to run")
parser.add_argument("--processes", action="store", type=int, default=1,
help="Number of simultaneous processes to use")
parser.add_argument("--include", action="append", type=slash_prefixed,
help="URL prefix to include")
parser.add_argument("--exclude", action="append", type=slash_prefixed,
help="URL prefix to exclude")
parser.add_argument("--include-manifest", type=abs_path,
help="Path to manifest listing tests to include")
parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--total-chunks", action="store", type=int, default=1,
help="Total number of chunks to use")
parser.add_argument("--this-chunk", action="store", type=int, default=1,
help="Chunk number to run")
parser.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"],
default=None, help="Chunking type to use")
parser.add_argument("--list-test-groups", action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
parser.add_argument("--list-disabled", action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument("--repeat", action="store", type=int, default=1,
help="Number of times to run the tests")
parser.add_argument("--no-capture-stdio", action="store_true", default=False,
help="Don't capture stdio and write to logging")
parser.add_argument("--product", action="store", choices=product_choices,
default="firefox", help="Browser against which to run tests")
parser.add_argument('--debugger',
help="run under a debugger, e.g. gdb or valgrind")
parser.add_argument('--debugger-args', help="arguments to the debugger")
parser.add_argument('--pause-on-unexpected', action="store_true",
help="Halt the test runner when an unexpected result is encountered")
parser.add_argument("--symbols-path", action="store", type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
parser.add_argument("--stackwalk-binary", action="store", type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
parser.add_argument("--b2g-no-backup", action="store_true", default=False,
help="Don't backup device before testrun with --product=b2g")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(
description="""Runner for web-platform-tests tests.""",
usage="""%(prog)s [OPTION]... [TEST]...
TEST is either the full path to a test file to run, or the URL of a test excluding
scheme host and port.""")
parser.add_argument("--manifest-update",
action="store_true",
default=None,
help="Regenerate the test manifest.")
parser.add_argument("--no-manifest-update",
action="store_false",
dest="manifest_update",
help="Prevent regeneration of the test manifest.")
parser.add_argument(
"--manifest-download",
action="store_true",
default=None,
help="Attempt to download a preexisting manifest when updating.")
parser.add_argument(
"--timeout-multiplier",
action="store",
type=float,
default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument(
"--run-by-dir",
type=int,
nargs="?",
default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--processes",
action="store",
type=int,
default=None,
help="Number of simultaneous processes to use")
parser.add_argument("--no-capture-stdio",
action="store_true",
default=False,
help="Don't capture stdio and write to logging")
parser.add_argument(
"--no-fail-on-unexpected",
action="store_false",
default=True,
dest="fail_on_unexpected",
help="Exit with status code 0 when test expectations are violated")
mode_group = parser.add_argument_group("Mode")
mode_group.add_argument(
"--list-test-groups",
action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
mode_group.add_argument(
"--list-disabled",
action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
mode_group.add_argument("--list-tests",
action="store_true",
default=False,
help="List all tests that will run")
stability_group = mode_group.add_mutually_exclusive_group()
stability_group.add_argument(
"--verify",
action="store_true",
default=False,
help="Run a stability check on the selected tests")
stability_group.add_argument("--stability",
action="store_true",
default=False,
help=argparse.SUPPRESS)
mode_group.add_argument(
"--verify-log-full",
action="store_true",
default=False,
help="Output per-iteration test results when running verify")
mode_group.add_argument(
"--verify-repeat-loop",
action="store",
default=10,
help=
"Number of iterations for a run that reloads each test without restart.",
type=int)
mode_group.add_argument(
"--verify-repeat-restart",
action="store",
default=5,
help=
"Number of iterations, for a run that restarts the runner between each iteration",
type=int)
chaos_mode_group = mode_group.add_mutually_exclusive_group()
chaos_mode_group.add_argument(
"--verify-no-chaos-mode",
action="store_false",
default=True,
dest="verify_chaos_mode",
help="Disable chaos mode when running on Firefox")
chaos_mode_group.add_argument(
"--verify-chaos-mode",
action="store_true",
default=True,
dest="verify_chaos_mode",
help="Enable chaos mode when running on Firefox")
mode_group.add_argument(
"--verify-max-time",
action="store",
default=None,
help="The maximum number of minutes for the job to run",
type=lambda x: timedelta(minutes=float(x)))
output_results_group = mode_group.add_mutually_exclusive_group()
output_results_group.add_argument(
"--verify-no-output-results",
action="store_false",
dest="verify_output_results",
default=True,
help="Prints individuals test results and messages")
output_results_group.add_argument(
"--verify-output-results",
action="store_true",
dest="verify_output_results",
default=True,
help="Disable printing individuals test results and messages")
test_selection_group = parser.add_argument_group("Test Selection")
test_selection_group.add_argument("--test-types",
action="store",
nargs="*",
default=wpttest.enabled_tests,
choices=wpttest.enabled_tests,
help="Test types to run")
test_selection_group.add_argument("--include",
action="append",
help="URL prefix to include")
test_selection_group.add_argument("--exclude",
action="append",
help="URL prefix to exclude")
test_selection_group.add_argument(
"--include-manifest",
type=abs_path,
help="Path to manifest listing tests to include")
test_selection_group.add_argument(
"--skip-timeout",
action="store_true",
help="Skip tests that are expected to time out")
test_selection_group.add_argument(
"--tag",
action="append",
dest="tags",
help="Labels applied to tests to include in the run. "
"Labels starting dir: are equivalent to top-level directories.")
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument(
'--debugger',
const="__default__",
nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args',
help="arguments to the debugger")
debugging_group.add_argument(
"--rerun",
action="store",
type=int,
default=1,
help="Number of times to re run each test without restarts")
debugging_group.add_argument(
"--repeat",
action="store",
type=int,
default=1,
help="Number of times to run the tests, restarting between each run")
debugging_group.add_argument(
"--repeat-until-unexpected",
action="store_true",
default=None,
help="Run tests in a loop until one returns an unexpected result")
debugging_group.add_argument(
'--pause-after-test',
action="store_true",
default=None,
help=
"Halt the test runner after each test (this happens by default if only a single test is run)"
)
debugging_group.add_argument(
'--no-pause-after-test',
dest="pause_after_test",
action="store_false",
help=
"Don't halt the test runner irrespective of the number of tests run")
debugging_group.add_argument(
'--pause-on-unexpected',
action="store_true",
help="Halt the test runner when an unexpected result is encountered")
debugging_group.add_argument('--no-restart-on-unexpected',
dest="restart_on_unexpected",
default=True,
action="store_false",
help="Don't restart on an unexpected result")
debugging_group.add_argument(
"--symbols-path",
action="store",
type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
debugging_group.add_argument(
"--stackwalk-binary",
action="store",
type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
debugging_group.add_argument("--pdb",
action="store_true",
help="Drop into pdb on python exception")
config_group = parser.add_argument_group("Configuration")
config_group.add_argument("--binary",
action="store",
type=abs_path,
help="Desktop binary to run tests against")
config_group.add_argument('--binary-arg',
default=[],
action="append",
dest="binary_args",
help="Extra argument for the binary")
config_group.add_argument("--webdriver-binary",
action="store",
metavar="BINARY",
type=abs_path,
help="WebDriver server binary to use")
config_group.add_argument('--webdriver-arg',
default=[],
action="append",
dest="webdriver_args",
help="Extra argument for the WebDriver binary")
config_group.add_argument("--package-name",
action="store",
help="Android package name to run tests against")
config_group.add_argument(
"--device-serial",
action="store",
help="Running Android instance to connect to, if not emulator-5554")
config_group.add_argument(
"--metadata",
action="store",
type=abs_path,
dest="metadata_root",
help="Path to root directory containing test metadata"),
config_group.add_argument(
"--tests",
action="store",
type=abs_path,
dest="tests_root",
help="Path to root directory containing test files"),
config_group.add_argument(
"--manifest",
action="store",
type=abs_path,
dest="manifest_path",
help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)"
)
config_group.add_argument(
"--run-info",
action="store",
type=abs_path,
help="Path to directory containing extra json files to add to run info"
)
config_group.add_argument("--product",
action="store",
choices=product_choices,
default=None,
help="Browser against which to run tests")
config_group.add_argument(
"--browser-version",
action="store",
default=None,
help="Informative string detailing the browser "
"release version. This is included in the run_info data.")
config_group.add_argument(
"--browser-channel",
action="store",
default=None,
help="Informative string detailing the browser "
"release channel. This is included in the run_info data.")
config_group.add_argument("--config",
action="store",
type=abs_path,
dest="config",
help="Path to config file")
config_group.add_argument(
"--install-fonts",
action="store_true",
default=None,
help="Allow the wptrunner to install fonts on your system")
config_group.add_argument("--font-dir",
action="store",
type=abs_path,
dest="font_dir",
help="Path to local font installation directory",
default=None)
config_group.add_argument("--headless",
action="store_true",
help="Run browser in headless mode",
default=None)
config_group.add_argument("--no-headless",
action="store_false",
dest="headless",
help="Don't run browser in headless mode")
build_type = parser.add_mutually_exclusive_group()
build_type.add_argument(
"--debug-build",
dest="debug",
action="store_true",
default=None,
help="Build is a debug build (overrides any mozinfo file)")
build_type.add_argument(
"--release-build",
dest="debug",
action="store_false",
default=None,
help="Build is a release (overrides any mozinfo file)")
chunking_group = parser.add_argument_group("Test Chunking")
chunking_group.add_argument("--total-chunks",
action="store",
type=int,
default=1,
help="Total number of chunks to use")
chunking_group.add_argument("--this-chunk",
action="store",
type=int,
default=1,
help="Chunk number to run")
chunking_group.add_argument(
"--chunk-type",
action="store",
choices=["none", "equal_time", "hash", "dir_hash"],
default=None,
help="Chunking type to use")
ssl_group = parser.add_argument_group("SSL/TLS")
ssl_group.add_argument(
"--ssl-type",
action="store",
default=None,
choices=["openssl", "pregenerated", "none"],
help=
"Type of ssl support to enable (running without ssl may lead to spurious errors)"
)
ssl_group.add_argument("--openssl-binary",
action="store",
help="Path to openssl binary",
default="openssl")
ssl_group.add_argument(
"--certutil-binary",
action="store",
help="Path to certutil binary for use with Firefox + ssl")
ssl_group.add_argument(
"--ca-cert-path",
action="store",
type=abs_path,
help="Path to ca certificate when using pregenerated ssl certificates")
ssl_group.add_argument(
"--host-key-path",
action="store",
type=abs_path,
help="Path to host private key when using pregenerated ssl certificates"
)
ssl_group.add_argument(
"--host-cert-path",
action="store",
type=abs_path,
help="Path to host certificate when using pregenerated ssl certificates"
)
gecko_group = parser.add_argument_group("Gecko-specific")
gecko_group.add_argument(
"--prefs-root",
dest="prefs_root",
action="store",
type=abs_path,
help="Path to the folder containing browser prefs")
gecko_group.add_argument("--disable-e10s",
dest="gecko_e10s",
action="store_false",
default=True,
help="Run tests without electrolysis preferences")
gecko_group.add_argument(
"--stackfix-dir",
dest="stackfix_dir",
action="store",
help="Path to directory containing assertion stack fixing scripts")
gecko_group.add_argument(
"--setpref",
dest="extra_prefs",
action='append',
default=[],
metavar="PREF=VALUE",
help="Defines an extra user preference (overrides those in prefs_root)"
)
gecko_group.add_argument(
"--leak-check",
dest="leak_check",
action="store_true",
default=None,
help="Enable leak checking (enabled by default for debug builds, "
"silently ignored for opt)")
gecko_group.add_argument("--no-leak-check",
dest="leak_check",
action="store_false",
default=None,
help="Disable leak checking")
gecko_group.add_argument(
"--stylo-threads",
action="store",
type=int,
default=1,
help="Number of parallel threads to use for stylo")
gecko_group.add_argument(
"--reftest-internal",
dest="reftest_internal",
action="store_true",
default=None,
help="Enable reftest runner implemented inside Marionette")
gecko_group.add_argument(
"--reftest-external",
dest="reftest_internal",
action="store_false",
help="Disable reftest runner implemented inside Marionette")
gecko_group.add_argument(
"--reftest-screenshot",
dest="reftest_screenshot",
action="store",
choices=["always", "fail", "unexpected"],
default="unexpected",
help="With --reftest-internal, when to take a screenshot")
gecko_group.add_argument(
"--chaos",
dest="chaos_mode_flags",
action="store",
nargs="?",
const=0xFFFFFFFF,
type=int,
help="Enable chaos mode with the specified feature flag "
"(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for "
"details). If no value is supplied, all features are activated")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument(
"--user-stylesheet",
default=[],
action="append",
dest="user_stylesheets",
help="Inject a user CSS stylesheet into every test.")
sauce_group = parser.add_argument_group("Sauce Labs-specific")
sauce_group.add_argument("--sauce-browser",
dest="sauce_browser",
help="Sauce Labs browser name")
sauce_group.add_argument("--sauce-platform",
dest="sauce_platform",
help="Sauce Labs OS platform")
sauce_group.add_argument("--sauce-version",
dest="sauce_version",
help="Sauce Labs browser version")
sauce_group.add_argument("--sauce-build",
dest="sauce_build",
help="Sauce Labs build identifier")
sauce_group.add_argument("--sauce-tags",
dest="sauce_tags",
nargs="*",
help="Sauce Labs identifying tag",
default=[])
sauce_group.add_argument("--sauce-tunnel-id",
dest="sauce_tunnel_id",
help="Sauce Connect tunnel identifier")
sauce_group.add_argument("--sauce-user",
dest="sauce_user",
help="Sauce Labs user name")
sauce_group.add_argument("--sauce-key",
dest="sauce_key",
default=os.environ.get("SAUCE_ACCESS_KEY"),
help="Sauce Labs access key")
sauce_group.add_argument("--sauce-connect-binary",
dest="sauce_connect_binary",
help="Path to Sauce Connect binary")
sauce_group.add_argument("--sauce-init-timeout",
action="store",
type=int,
default=30,
help="Number of seconds to wait for Sauce "
"Connect tunnel to be available before "
"aborting")
sauce_group.add_argument("--sauce-connect-arg",
action="append",
default=[],
dest="sauce_connect_args",
help="Command-line argument to forward to the "
"Sauce Connect binary (repeatable)")
webkit_group = parser.add_argument_group("WebKit-specific")
webkit_group.add_argument("--webkit-port",
dest="webkit_port",
help="WebKit port")
parser.add_argument(
"test_list",
nargs="*",
help="List of URLs for tests to run, or paths including tests to run. "
"(equivalent to --include)")
commandline.log_formatters["wptreport"] = (formatters.WptreportFormatter,
"wptreport format")
commandline.add_logging_group(parser)
return parser
def create_parser_update(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(
"web-platform-tests-update",
description="Update script for web-platform-tests tests.")
parser.add_argument("--product",
action="store",
choices=product_choices,
default=None,
help="Browser for which metadata is being updated")
parser.add_argument("--config",
action="store",
type=abs_path,
help="Path to config file")
parser.add_argument("--metadata",
action="store",
type=abs_path,
dest="metadata_root",
help="Path to the folder containing test metadata"),
parser.add_argument("--tests",
action="store",
type=abs_path,
dest="tests_root",
help="Path to web-platform-tests"),
parser.add_argument(
"--sync-path",
action="store",
type=abs_path,
help="Path to store git checkout of web-platform-tests during update"),
parser.add_argument(
"--remote_url",
action="store",
help="URL of web-platfrom-tests repository to sync against"),
parser.add_argument("--branch",
action="store",
type=abs_path,
help="Remote branch to sync against")
parser.add_argument("--rev", action="store", help="Revision to sync to")
parser.add_argument(
"--no-patch",
action="store_true",
help="Don't create an mq patch or git commit containing the changes.")
parser.add_argument("--sync",
dest="sync",
action="store_true",
default=False,
help="Sync the tests with the latest from upstream")
parser.add_argument(
"--ignore-existing",
action="store_true",
help=
"When updating test results only consider results from the logfiles provided, not existing expectations."
)
parser.add_argument(
"--continue",
action="store_true",
help="Continue a previously started run of the update script")
parser.add_argument(
"--abort",
action="store_true",
help="Clear state from a previous incomplete run of the update script")
# Should make this required iff run=logfile
parser.add_argument("run_log",
nargs="*",
type=abs_path,
help="Log file from run of tests")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(
description="""Runner for web-platform-tests tests.""",
usage="""%(prog)s [OPTION]... [TEST]...
TEST is either the full path to a test file to run, or the URL of a test excluding
scheme host and port.""")
parser.add_argument("--manifest-update",
action="store_true",
default=False,
help="Regenerate the test manifest.")
parser.add_argument(
"--timeout-multiplier",
action="store",
type=float,
default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument(
"--run-by-dir",
type=int,
nargs="?",
default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--processes",
action="store",
type=int,
default=None,
help="Number of simultaneous processes to use")
parser.add_argument("--no-capture-stdio",
action="store_true",
default=False,
help="Don't capture stdio and write to logging")
mode_group = parser.add_argument_group("Mode")
mode_group.add_argument(
"--list-test-groups",
action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
mode_group.add_argument(
"--list-disabled",
action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
test_selection_group = parser.add_argument_group("Test Selection")
test_selection_group.add_argument("--test-types",
action="store",
nargs="*",
default=wpttest.enabled_tests,
choices=wpttest.enabled_tests,
help="Test types to run")
test_selection_group.add_argument("--include",
action="append",
help="URL prefix to include")
test_selection_group.add_argument("--exclude",
action="append",
help="URL prefix to exclude")
test_selection_group.add_argument(
"--include-manifest",
type=abs_path,
help="Path to manifest listing tests to include")
test_selection_group.add_argument(
"--tag",
action="append",
dest="tags",
help=
"Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories."
)
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument(
'--debugger',
const="__default__",
nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args',
help="arguments to the debugger")
debugging_group.add_argument("--repeat",
action="store",
type=int,
default=1,
help="Number of times to run the tests")
debugging_group.add_argument(
"--repeat-until-unexpected",
action="store_true",
default=None,
help="Run tests in a loop until one returns an unexpected result")
debugging_group.add_argument(
'--pause-after-test',
action="store_true",
default=None,
help=
"Halt the test runner after each test (this happens by default if only a single test is run)"
)
debugging_group.add_argument(
'--no-pause-after-test',
dest="pause_after_test",
action="store_false",
help=
"Don't halt the test runner irrespective of the number of tests run")
debugging_group.add_argument(
'--pause-on-unexpected',
action="store_true",
help="Halt the test runner when an unexpected result is encountered")
debugging_group.add_argument(
"--symbols-path",
action="store",
type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
debugging_group.add_argument(
"--stackwalk-binary",
action="store",
type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
debugging_group.add_argument("--pdb",
action="store_true",
help="Drop into pdb on python exception")
config_group = parser.add_argument_group("Configuration")
config_group.add_argument("--binary",
action="store",
type=abs_path,
help="Binary to run tests against")
config_group.add_argument('--binary-arg',
default=[],
action="append",
dest="binary_args",
help="Extra argument for the binary (servo)")
config_group.add_argument("--webdriver-binary",
action="store",
metavar="BINARY",
type=abs_path,
help="WebDriver server binary to use")
config_group.add_argument(
"--metadata",
action="store",
type=abs_path,
dest="metadata_root",
help="Path to root directory containing test metadata"),
config_group.add_argument(
"--tests",
action="store",
type=abs_path,
dest="tests_root",
help="Path to root directory containing test files"),
config_group.add_argument(
"--run-info",
action="store",
type=abs_path,
help="Path to directory containing extra json files to add to run info"
)
config_group.add_argument("--product",
action="store",
choices=product_choices,
default=None,
help="Browser against which to run tests")
config_group.add_argument("--config",
action="store",
type=abs_path,
dest="config",
help="Path to config file")
build_type = parser.add_mutually_exclusive_group()
build_type.add_argument(
"--debug-build",
dest="debug",
action="store_true",
default=None,
help="Build is a debug build (overrides any mozinfo file)")
build_type.add_argument(
"--release-build",
dest="debug",
action="store_false",
default=None,
help="Build is a release (overrides any mozinfo file)")
chunking_group = parser.add_argument_group("Test Chunking")
chunking_group.add_argument("--total-chunks",
action="store",
type=int,
default=1,
help="Total number of chunks to use")
chunking_group.add_argument("--this-chunk",
action="store",
type=int,
default=1,
help="Chunk number to run")
chunking_group.add_argument(
"--chunk-type",
action="store",
choices=["none", "equal_time", "hash", "dir_hash"],
default=None,
help="Chunking type to use")
ssl_group = parser.add_argument_group("SSL/TLS")
ssl_group.add_argument(
"--ssl-type",
action="store",
default=None,
choices=["openssl", "pregenerated", "none"],
help=
"Type of ssl support to enable (running without ssl may lead to spurious errors)"
)
ssl_group.add_argument("--openssl-binary",
action="store",
help="Path to openssl binary",
default="openssl")
ssl_group.add_argument(
"--certutil-binary",
action="store",
help="Path to certutil binary for use with Firefox + ssl")
ssl_group.add_argument(
"--ca-cert-path",
action="store",
type=abs_path,
help="Path to ca certificate when using pregenerated ssl certificates")
ssl_group.add_argument(
"--host-key-path",
action="store",
type=abs_path,
help="Path to host private key when using pregenerated ssl certificates"
)
ssl_group.add_argument(
"--host-cert-path",
action="store",
type=abs_path,
help="Path to host certificate when using pregenerated ssl certificates"
)
gecko_group = parser.add_argument_group("Gecko-specific")
gecko_group.add_argument(
"--prefs-root",
dest="prefs_root",
action="store",
type=abs_path,
help="Path to the folder containing browser prefs")
gecko_group.add_argument("--disable-e10s",
dest="gecko_e10s",
action="store_false",
default=True,
help="Run tests without electrolysis preferences")
b2g_group = parser.add_argument_group("B2G-specific")
b2g_group.add_argument(
"--b2g-no-backup",
action="store_true",
default=False,
help="Don't backup device before testrun with --product=b2g")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument(
"--user-stylesheet",
default=[],
action="append",
dest="user_stylesheets",
help="Inject a user CSS stylesheet into every test.")
servo_group.add_argument("--servo-backend",
default="cpu",
choices=["cpu", "webrender"],
help="Rendering backend to use with Servo.")
parser.add_argument(
"test_list",
nargs="*",
help="List of URLs for tests to run, or paths including tests to run. "
"(equivalent to --include)")
commandline.add_logging_group(parser)
return parser
def create_parser_update(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(
"web-platform-tests-update",
description="Update script for web-platform-tests tests.")
parser.add_argument("--product",
action="store",
choices=product_choices,
default=None,
help="Browser for which metadata is being updated")
parser.add_argument("--config",
action="store",
type=abs_path,
help="Path to config file")
parser.add_argument("--metadata",
action="store",
type=abs_path,
dest="metadata_root",
help="Path to the folder containing test metadata"),
parser.add_argument("--tests",
action="store",
type=abs_path,
dest="tests_root",
help="Path to web-platform-tests"),
parser.add_argument(
"--manifest",
action="store",
type=abs_path,
dest="manifest_path",
help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)"
)
parser.add_argument(
"--sync-path",
action="store",
type=abs_path,
help="Path to store git checkout of web-platform-tests during update"),
parser.add_argument(
"--remote_url",
action="store",
help="URL of web-platfrom-tests repository to sync against"),
parser.add_argument("--branch",
action="store",
type=abs_path,
help="Remote branch to sync against")
parser.add_argument("--rev", action="store", help="Revision to sync to")
parser.add_argument("--patch",
action="store_true",
dest="patch",
default=None,
help="Create a VCS commit containing the changes.")
parser.add_argument(
"--no-patch",
action="store_false",
dest="patch",
help="Don't create a VCS commit containing the changes.")
parser.add_argument(
"--sync",
dest="sync",
action="store_true",
default=False,
help="Sync the tests with the latest from upstream (implies --patch)")
parser.add_argument(
"--ignore-existing",
action="store_true",
help=
"When updating test results only consider results from the logfiles provided, not existing expectations."
)
parser.add_argument(
"--stability",
nargs="?",
action="store",
const="unstable",
default=None,
help=
("Reason for disabling tests. When updating test results, disable tests that have "
"inconsistent results across many runs with the given reason."))
parser.add_argument(
"--no-remove-obsolete",
action="store_false",
dest="remove_obsolete",
default=True,
help=
("Don't remove metadata files that no longer correspond to a test file"
))
parser.add_argument(
"--no-store-state",
action="store_false",
dest="store_state",
help="Store state so that steps can be resumed after failure")
parser.add_argument(
"--continue",
action="store_true",
help="Continue a previously started run of the update script")
parser.add_argument(
"--abort",
action="store_true",
help="Clear state from a previous incomplete run of the update script")
parser.add_argument(
"--exclude",
action="store",
nargs="*",
help="List of glob-style paths to exclude when syncing tests")
parser.add_argument(
"--include",
action="store",
nargs="*",
help=
"List of glob-style paths to include which would otherwise be excluded when syncing tests"
)
parser.add_argument(
"--extra-property",
action="append",
default=[],
help="Extra property from run_info.json to use in metadata update")
# Should make this required iff run=logfile
parser.add_argument("run_log",
nargs="*",
type=abs_path,
help="Log file from run of tests")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.")
parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
help="Path to the folder containing test metadata"),
parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
help="Path to test files"),
parser.add_argument("--run-info", action="store", type=abs_path,
help="Path to directory containing extra json files to add to run info")
parser.add_argument("--config", action="store", type=abs_path, dest="config",
help="Path to config file")
parser.add_argument("--manifest-update", action="store_true", default=False,
help="Force regeneration of the test manifest")
parser.add_argument("--binary", action="store",
type=abs_path, help="Binary to run tests against")
parser.add_argument("--webdriver-binary", action="store", metavar="BINARY",
type=abs_path, help="WebDriver server binary to use")
parser.add_argument("--processes", action="store", type=int, default=None,
help="Number of simultaneous processes to use")
parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument("--repeat", action="store", type=int, default=1,
help="Number of times to run the tests")
parser.add_argument("--no-capture-stdio", action="store_true", default=False,
help="Don't capture stdio and write to logging")
parser.add_argument("--product", action="store", choices=product_choices,
default=None, help="Browser against which to run tests")
parser.add_argument("--list-test-groups", action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
parser.add_argument("--list-disabled", action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
test_selection_group = parser.add_argument_group("Test Selection")
test_selection_group.add_argument("--test-types", action="store",
nargs="*", default=["testharness", "reftest"],
choices=["testharness", "reftest"],
help="Test types to run")
test_selection_group.add_argument("--include", action="append",
help="URL prefix to include")
test_selection_group.add_argument("--exclude", action="append",
help="URL prefix to exclude")
test_selection_group.add_argument("--include-manifest", type=abs_path,
help="Path to manifest listing tests to include")
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument('--debugger', const="__default__", nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
help="Halt the test runner after each test (this happens by default if only a single test is run)")
debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
help="Don't halt the test runner irrespective of the number of tests run")
debugging_group.add_argument('--pause-on-unexpected', action="store_true",
help="Halt the test runner when an unexpected result is encountered")
debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
chunking_group = parser.add_argument_group("Test Chunking")
chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
help="Total number of chunks to use")
chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
help="Chunk number to run")
chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"],
default=None, help="Chunking type to use")
ssl_group = parser.add_argument_group("SSL/TLS")
ssl_group.add_argument("--ssl-type", action="store", default=None,
choices=["openssl", "pregenerated", "none"],
help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
ssl_group.add_argument("--openssl-binary", action="store",
help="Path to openssl binary", default="openssl")
ssl_group.add_argument("--certutil-binary", action="store",
help="Path to certutil binary for use with Firefox + ssl")
ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path,
help="Path to ca certificate when using pregenerated ssl certificates")
ssl_group.add_argument("--host-key-path", action="store", type=abs_path,
help="Path to host private key when using pregenerated ssl certificates")
ssl_group.add_argument("--host-cert-path", action="store", type=abs_path,
help="Path to host certificate when using pregenerated ssl certificates")
gecko_group = parser.add_argument_group("Gecko-specific")
gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
help="Path to the folder containing browser prefs")
b2g_group = parser.add_argument_group("B2G-specific")
b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False,
help="Don't backup device before testrun with --product=b2g")
parser.add_argument("test_list", nargs="*",
help="List of URLs for tests to run, or paths including tests to run. "
"(equivalent to --include)")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""",
usage="""%(prog)s [OPTION]... [TEST]...
TEST is either the full path to a test file to run, or the URL of a test excluding
scheme host and port.""")
parser.add_argument("--manifest-update", action="store_true", default=False,
help="Regenerate the test manifest.")
parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--processes", action="store", type=int, default=None,
help="Number of simultaneous processes to use")
parser.add_argument("--no-capture-stdio", action="store_true", default=False,
help="Don't capture stdio and write to logging")
mode_group = parser.add_argument_group("Mode")
mode_group.add_argument("--list-test-groups", action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
mode_group.add_argument("--list-disabled", action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
test_selection_group = parser.add_argument_group("Test Selection")
test_selection_group.add_argument("--test-types", action="store",
nargs="*", default=wpttest.enabled_tests,
choices=wpttest.enabled_tests,
help="Test types to run")
test_selection_group.add_argument("--include", action="append",
help="URL prefix to include")
test_selection_group.add_argument("--exclude", action="append",
help="URL prefix to exclude")
test_selection_group.add_argument("--include-manifest", type=abs_path,
help="Path to manifest listing tests to include")
test_selection_group.add_argument("--tag", action="append", dest="tags",
help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.")
debugging_group = parser.add_argument_group("Debugging")
debugging_group.add_argument('--debugger', const="__default__", nargs="?",
help="run under a debugger, e.g. gdb or valgrind")
debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
debugging_group.add_argument("--repeat", action="store", type=int, default=1,
help="Number of times to run the tests")
debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None,
help="Run tests in a loop until one returns an unexpected result")
debugging_group.add_argument('--pause-after-test', action="store_true", default=None,
help="Halt the test runner after each test (this happens by default if only a single test is run)")
debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false",
help="Don't halt the test runner irrespective of the number of tests run")
debugging_group.add_argument('--pause-on-unexpected', action="store_true",
help="Halt the test runner when an unexpected result is encountered")
debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected",
default=True, action="store_false",
help="Don't restart on an unexpected result")
debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
debugging_group.add_argument("--pdb", action="store_true",
help="Drop into pdb on python exception")
config_group = parser.add_argument_group("Configuration")
config_group.add_argument("--binary", action="store",
type=abs_path, help="Binary to run tests against")
config_group.add_argument('--binary-arg',
default=[], action="append", dest="binary_args",
help="Extra argument for the binary")
config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY",
type=abs_path, help="WebDriver server binary to use")
config_group.add_argument('--webdriver-arg',
default=[], action="append", dest="webdriver_args",
help="Extra argument for the WebDriver binary")
config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
help="Path to root directory containing test metadata"),
config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
help="Path to root directory containing test files"),
config_group.add_argument("--run-info", action="store", type=abs_path,
help="Path to directory containing extra json files to add to run info")
config_group.add_argument("--product", action="store", choices=product_choices,
default=None, help="Browser against which to run tests")
config_group.add_argument("--config", action="store", type=abs_path, dest="config",
help="Path to config file")
build_type = parser.add_mutually_exclusive_group()
build_type.add_argument("--debug-build", dest="debug", action="store_true",
default=None,
help="Build is a debug build (overrides any mozinfo file)")
build_type.add_argument("--release-build", dest="debug", action="store_false",
default=None,
help="Build is a release (overrides any mozinfo file)")
chunking_group = parser.add_argument_group("Test Chunking")
chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
help="Total number of chunks to use")
chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
help="Chunk number to run")
chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"],
default=None, help="Chunking type to use")
ssl_group = parser.add_argument_group("SSL/TLS")
ssl_group.add_argument("--ssl-type", action="store", default=None,
choices=["openssl", "pregenerated", "none"],
help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
ssl_group.add_argument("--openssl-binary", action="store",
help="Path to openssl binary", default="openssl")
ssl_group.add_argument("--certutil-binary", action="store",
help="Path to certutil binary for use with Firefox + ssl")
ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path,
help="Path to ca certificate when using pregenerated ssl certificates")
ssl_group.add_argument("--host-key-path", action="store", type=abs_path,
help="Path to host private key when using pregenerated ssl certificates")
ssl_group.add_argument("--host-cert-path", action="store", type=abs_path,
help="Path to host certificate when using pregenerated ssl certificates")
gecko_group = parser.add_argument_group("Gecko-specific")
gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
help="Path to the folder containing browser prefs")
gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
help="Run tests without electrolysis preferences")
gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store",
help="Path to directory containing assertion stack fixing scripts")
gecko_group.add_argument("--setpref", dest="extra_prefs", action='append',
default=[], metavar="PREF=VALUE",
help="Defines an extra user preference (overrides those in prefs_root)")
gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true",
help="Enable leak checking")
servo_group = parser.add_argument_group("Servo-specific")
servo_group.add_argument("--user-stylesheet",
default=[], action="append", dest="user_stylesheets",
help="Inject a user CSS stylesheet into every test.")
sauce_group = parser.add_argument_group("Sauce Labs-specific")
sauce_group.add_argument("--sauce-browser", dest="sauce_browser",
help="Sauce Labs browser name")
sauce_group.add_argument("--sauce-platform", dest="sauce_platform",
help="Sauce Labs OS platform")
sauce_group.add_argument("--sauce-version", dest="sauce_version",
help="Sauce Labs browser version")
sauce_group.add_argument("--sauce-build", dest="sauce_build",
help="Sauce Labs build identifier")
sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*",
help="Sauce Labs identifying tag", default=[])
sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id",
help="Sauce Connect tunnel identifier")
sauce_group.add_argument("--sauce-user", dest="sauce_user",
help="Sauce Labs user name")
sauce_group.add_argument("--sauce-key", dest="sauce_key",
default=os.environ.get("SAUCE_ACCESS_KEY"),
help="Sauce Labs access key")
sauce_group.add_argument("--sauce-connect-binary",
dest="sauce_connect_binary",
help="Path to Sauce Connect binary")
parser.add_argument("test_list", nargs="*",
help="List of URLs for tests to run, or paths including tests to run. "
"(equivalent to --include)")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.")
parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
help="Path to the folder containing test metadata"),
parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root",
help="Path to test files"),
parser.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
help="Path to the folder containing browser prefs"),
parser.add_argument("--serve-root", action="store", type=abs_path, dest="serve_root",
help="Path to web-platform-tests checkout containing serve.py and manifest.py"
" (defaults to test_root)")
parser.add_argument("--run-info", action="store", type=abs_path,
help="Path to directory containing extra json files to add to run info")
parser.add_argument("--config", action="store", type=abs_path, dest="config",
help="Path to config file")
parser.add_argument("--manifest-update", action="store_true", default=False,
help="Force regeneration of the test manifest")
parser.add_argument("--binary", action="store",
type=abs_path, help="Binary to run tests against")
parser.add_argument("--test-types", action="store",
nargs="*", default=["testharness", "reftest"],
choices=["testharness", "reftest"],
help="Test types to run")
parser.add_argument("--processes", action="store", type=int, default=1,
help="Number of simultaneous processes to use")
parser.add_argument("--include", action="append", type=slash_prefixed,
help="URL prefix to include")
parser.add_argument("--exclude", action="append", type=slash_prefixed,
help="URL prefix to exclude")
parser.add_argument("--include-manifest", type=abs_path,
help="Path to manifest listing tests to include")
parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory")
parser.add_argument("--total-chunks", action="store", type=int, default=1,
help="Total number of chunks to use")
parser.add_argument("--this-chunk", action="store", type=int, default=1,
help="Chunk number to run")
parser.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"],
default=None, help="Chunking type to use")
parser.add_argument("--list-test-groups", action="store_true",
default=False,
help="List the top level directories containing tests that will run.")
parser.add_argument("--list-disabled", action="store_true",
default=False,
help="List the tests that are disabled on the current platform")
parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
help="Multiplier relative to standard test timeout to use")
parser.add_argument("--repeat", action="store", type=int, default=1,
help="Number of times to run the tests")
parser.add_argument("--no-capture-stdio", action="store_true", default=False,
help="Don't capture stdio and write to logging")
parser.add_argument("--product", action="store", choices=product_choices,
default="firefox", help="Browser against which to run tests")
parser.add_argument('--debugger',
help="run under a debugger, e.g. gdb or valgrind")
parser.add_argument('--debugger-args', help="arguments to the debugger")
parser.add_argument('--pause-on-unexpected', action="store_true",
help="Halt the test runner when an unexpected result is encountered")
parser.add_argument("--symbols-path", action="store", type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.")
parser.add_argument("--stackwalk-binary", action="store", type=abs_path,
help="Path to stackwalker program used to analyse minidumps.")
parser.add_argument("--b2g-no-backup", action="store_true", default=False,
help="Don't backup device before testrun with --product=b2g")
commandline.add_logging_group(parser)
return parser
def create_parser(product_choices=None):
from mozlog.structured import commandline
import products
if product_choices is None:
config_data = config.load()
product_choices = products.products_enabled(config_data)
parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.")
parser.add_argument(
"--metadata",
action="store",
type=abs_path,
dest="metadata_root",
help="Path to the folder containing test metadata",
),
parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to test files"),
parser.add_argument(
"--prefs-root",
dest="prefs_root",
action="store",
type=abs_path,
help="Path to the folder containing browser prefs",
),
parser.add_argument(
"--serve-root",
action="store",
type=abs_path,
dest="serve_root",
help="Path to web-platform-tests checkout containing serve.py and manifest.py" " (defaults to test_root)",
)
parser.add_argument(
"--run-info",
action="store",
type=abs_path,
help="Path to directory containing extra json files to add to run info",
)
parser.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file")
parser.add_argument(
"--manifest-update", action="store_true", default=False, help="Force regeneration of the test manifest"
)
parser.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against")
parser.add_argument(
"--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use"
)
parser.add_argument(
"--test-types",
action="store",
nargs="*",
default=["testharness", "reftest"],
choices=["testharness", "reftest"],
help="Test types to run",
)
parser.add_argument(
"--processes", action="store", type=int, default=1, help="Number of simultaneous processes to use"
)
parser.add_argument("--include", action="append", type=slash_prefixed, help="URL prefix to include")
parser.add_argument("--exclude", action="append", type=slash_prefixed, help="URL prefix to exclude")
parser.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include")
parser.add_argument(
"--run-by-dir",
type=int,
nargs="?",
default=False,
help="Split run into groups by directories. With a parameter,"
"limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
"directory",
)
parser.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use")
parser.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run")
parser.add_argument(
"--chunk-type",
action="store",
choices=["none", "equal_time", "hash"],
default=None,
help="Chunking type to use",
)
parser.add_argument(
"--list-test-groups",
action="store_true",
default=False,
help="List the top level directories containing tests that will run.",
)
parser.add_argument(
"--list-disabled",
action="store_true",
default=False,
help="List the tests that are disabled on the current platform",
)
parser.add_argument(
"--timeout-multiplier",
action="store",
type=float,
default=None,
help="Multiplier relative to standard test timeout to use",
)
parser.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests")
parser.add_argument(
"--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging"
)
parser.add_argument(
"--product",
action="store",
choices=product_choices,
default="firefox",
help="Browser against which to run tests",
)
parser.add_argument("--debugger", help="run under a debugger, e.g. gdb or valgrind")
parser.add_argument("--debugger-args", help="arguments to the debugger")
parser.add_argument(
"--pause-on-unexpected",
action="store_true",
help="Halt the test runner when an unexpected result is encountered",
)
parser.add_argument(
"--symbols-path",
action="store",
type=url_or_path,
help="Path or url to symbols file used to analyse crash minidumps.",
)
parser.add_argument(
"--stackwalk-binary",
action="store",
type=abs_path,
help="Path to stackwalker program used to analyse minidumps.",
)
parser.add_argument(
"--ssl-type",
action="store",
default=None,
choices=["openssl", "pregenerated", "none"],
help="Type of ssl support to enable (running without ssl may lead to spurious errors)",
)
parser.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl")
parser.add_argument("--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl")
parser.add_argument(
"--ca-cert-path",
action="store",
type=abs_path,
help="Path to ca certificate when using pregenerated ssl certificates",
)
parser.add_argument(
"--host-key-path",
action="store",
type=abs_path,
help="Path to host private key when using pregenerated ssl certificates",
)
parser.add_argument(
"--host-cert-path",
action="store",
type=abs_path,
help="Path to host certificate when using pregenerated ssl certificates",
)
parser.add_argument(
"--b2g-no-backup",
action="store_true",
default=False,
help="Don't backup device before testrun with --product=b2g",
)
commandline.add_logging_group(parser)
return parser