def create_parser_update(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser( "web-platform-tests-update", description="Update script for web-platform-tests tests." ) parser.add_argument( "--product", action="store", choices=product_choices, default=None, help="Browser for which metadata is being updated", ) parser.add_argument("--config", action="store", type=abs_path, help="Path to config file") parser.add_argument( "--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata", ), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to web-platform-tests"), parser.add_argument( "--sync-path", action="store", type=abs_path, help="Path to store git checkout of web-platform-tests during update", ), parser.add_argument("--remote_url", action="store", help="URL of web-platfrom-tests repository to sync against"), parser.add_argument("--branch", action="store", type=abs_path, help="Remote branch to sync against") parser.add_argument("--rev", action="store", help="Revision to sync to") parser.add_argument( "--no-patch", action="store_true", help="Don't create an mq patch or git commit containing the changes." ) parser.add_argument( "--sync", dest="sync", action="store_true", default=False, help="Sync the tests with the latest from upstream" ) parser.add_argument( "--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.", ) parser.add_argument( "--continue", action="store_true", help="Continue a previously started run of the update script" ) parser.add_argument( "--abort", action="store_true", help="Clear state from a previous incomplete run of the update script" ) # Should make this required iff run=logfile parser.add_argument("run_log", nargs="*", type=abs_path, help="Log file from run of tests") commandline.add_logging_group(parser) return parser
def create_parser_update(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser("web-platform-tests-update", description="Update script for web-platform-tests tests.") parser.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser for which metadata is being updated") parser.add_argument("--config", action="store", type=abs_path, help="Path to config file") parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata"), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to web-platform-tests"), parser.add_argument("--manifest", action="store", type=abs_path, dest="manifest_path", help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)") parser.add_argument("--sync-path", action="store", type=abs_path, help="Path to store git checkout of web-platform-tests during update"), parser.add_argument("--remote_url", action="store", help="URL of web-platfrom-tests repository to sync against"), parser.add_argument("--branch", action="store", type=abs_path, help="Remote branch to sync against") parser.add_argument("--rev", action="store", help="Revision to sync to") parser.add_argument("--patch", action="store_true", dest="patch", default=None, help="Create a VCS commit containing the changes.") parser.add_argument("--no-patch", action="store_false", dest="patch", help="Don't create a VCS commit containing the changes.") parser.add_argument("--sync", dest="sync", action="store_true", default=False, help="Sync the tests with the latest from upstream (implies --patch)") parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.") parser.add_argument("--stability", nargs="?", action="store", const="unstable", default=None, help=("Reason for disabling tests. When updating test results, disable tests that have " "inconsistent results across many runs with the given reason.")) parser.add_argument("--no-remove-obsolete", action="store_false", dest="remove_obsolete", default=True, help=("Don't remove metadata files that no longer correspond to a test file")) parser.add_argument("--no-store-state", action="store_false", dest="store_state", help="Store state so that steps can be resumed after failure") parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script") parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script") parser.add_argument("--exclude", action="store", nargs="*", help="List of glob-style paths to exclude when syncing tests") parser.add_argument("--include", action="store", nargs="*", help="List of glob-style paths to include which would otherwise be excluded when syncing tests") parser.add_argument("--extra-property", action="append", default=[], help="Extra property from run_info.json to use in metadata update") # Should make this required iff run=logfile parser.add_argument("run_log", nargs="*", type=abs_path, help="Log file from run of tests") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.") parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata"), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to test files"), parser.add_argument("--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info") parser.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") parser.add_argument("--manifest-update", action="store_true", default=False, help="Force regeneration of the test manifest") parser.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") parser.add_argument("--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use") parser.add_argument("--processes", action="store", type=int, default=None, help="Number of simultaneous processes to use") parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") parser.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser against which to run tests") parser.add_argument("--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") parser.add_argument("--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") build_type = parser.add_mutually_exclusive_group() build_type.add_argument("--debug-build", dest="debug", action="store_true", default=None, help="Build is a debug build (overrides any mozinfo file)") build_type.add_argument("--release-build", dest="debug", action="store_false", default=None, help="Build is a release (overrides any mozinfo file)") test_selection_group = parser.add_argument_group("Test Selection") test_selection_group.add_argument("--test-types", action="store", nargs="*", default=["testharness", "reftest"], choices=["testharness", "reftest"], help="Test types to run") test_selection_group.add_argument("--include", action="append", help="URL prefix to include") test_selection_group.add_argument("--exclude", action="append", help="URL prefix to exclude") test_selection_group.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") test_selection_group.add_argument("--tag", action="append", dest="tags", help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.") debugging_group = parser.add_argument_group("Debugging") debugging_group.add_argument('--debugger', const="__default__", nargs="?", help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") debugging_group.add_argument('--pause-after-test', action="store_true", default=None, help="Halt the test runner after each test (this happens by default if only a single test is run)") debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false", help="Don't halt the test runner irrespective of the number of tests run") debugging_group.add_argument('--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") debugging_group.add_argument("--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") chunking_group = parser.add_argument_group("Test Chunking") chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"], default=None, help="Chunking type to use") ssl_group = parser.add_argument_group("SSL/TLS") ssl_group.add_argument("--ssl-type", action="store", default=None, choices=["openssl", "pregenerated", "none"], help="Type of ssl support to enable (running without ssl may lead to spurious errors)") ssl_group.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl") ssl_group.add_argument("--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl") ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path, help="Path to ca certificate when using pregenerated ssl certificates") ssl_group.add_argument("--host-key-path", action="store", type=abs_path, help="Path to host private key when using pregenerated ssl certificates") ssl_group.add_argument("--host-cert-path", action="store", type=abs_path, help="Path to host certificate when using pregenerated ssl certificates") gecko_group = parser.add_argument_group("Gecko-specific") gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs") b2g_group = parser.add_argument_group("B2G-specific") b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False, help="Don't backup device before testrun with --product=b2g") servo_group = parser.add_argument_group("Servo-specific") servo_group.add_argument("--user-stylesheet", default=[], action="append", dest="user_stylesheets", help="Inject a user CSS stylesheet into every test.") parser.add_argument("test_list", nargs="*", help="List of URLs for tests to run, or paths including tests to run. " "(equivalent to --include)") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""", usage="""%(prog)s [OPTION]... [TEST]... TEST is either the full path to a test file to run, or the URL of a test excluding scheme host and port.""") parser.add_argument("--manifest-update", action="store_true", default=None, help="Regenerate the test manifest.") parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update", help="Prevent regeneration of the test manifest.") parser.add_argument("--manifest-download", action="store_true", default=None, help="Attempt to download a preexisting manifest when updating.") parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--processes", action="store", type=int, default=None, help="Number of simultaneous processes to use") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") parser.add_argument("--no-fail-on-unexpected", action="store_false", default=True, dest="fail_on_unexpected", help="Exit with status code 0 when test expectations are violated") mode_group = parser.add_argument_group("Mode") mode_group.add_argument("--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") mode_group.add_argument("--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") mode_group.add_argument("--list-tests", action="store_true", default=False, help="List all tests that will run") mode_group.add_argument("--verify", action="store_true", default=False, help="Run a stability check on the selected tests") mode_group.add_argument("--verify-log-full", action="store_true", default=False, help="Output per-iteration test results when running verify") mode_group.add_argument("--verify-repeat-loop", action="store", default=10, help="Number of iterations for a run that reloads each test without restart.", type=int) mode_group.add_argument("--verify-repeat-restart", action="store", default=5, help="Number of iterations, for a run that restarts the runner between each iteration", type=int) chaos_mode_group = mode_group.add_mutually_exclusive_group() chaos_mode_group.add_argument("--verify-no-chaos-mode", action="store_false", default=True, dest="verify_chaos_mode", help="Disable chaos mode when running on Firefox") chaos_mode_group.add_argument("--verify-chaos-mode", action="store_true", default=True, dest="verify_chaos_mode", help="Enable chaos mode when running on Firefox") mode_group.add_argument("--verify-max-time", action="store", default=None, help="The maximum number of minutes for the job to run", type=lambda x: timedelta(minutes=float(x))) output_results_group = mode_group.add_mutually_exclusive_group() output_results_group.add_argument("--verify-no-output-results", action="store_false", dest="verify_output_results", default=True, help="Prints individuals test results and messages") output_results_group.add_argument("--verify-output-results", action="store_true", dest="verify_output_results", default=True, help="Disable printing individuals test results and messages") test_selection_group = parser.add_argument_group("Test Selection") test_selection_group.add_argument("--test-types", action="store", nargs="*", default=wpttest.enabled_tests, choices=wpttest.enabled_tests, help="Test types to run") test_selection_group.add_argument("--include", action="append", help="URL prefix to include") test_selection_group.add_argument("--exclude", action="append", help="URL prefix to exclude") test_selection_group.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") test_selection_group.add_argument("--skip-timeout", action="store_true", help="Skip tests that are expected to time out") test_selection_group.add_argument("--tag", action="append", dest="tags", help="Labels applied to tests to include in the run. " "Labels starting dir: are equivalent to top-level directories.") debugging_group = parser.add_argument_group("Debugging") debugging_group.add_argument('--debugger', const="__default__", nargs="?", help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") debugging_group.add_argument("--rerun", action="store", type=int, default=1, help="Number of times to re run each test without restarts") debugging_group.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests, restarting between each run") debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None, help="Run tests in a loop until one returns an unexpected result") debugging_group.add_argument('--pause-after-test', action="store_true", default=None, help="Halt the test runner after each test (this happens by default if only a single test is run)") debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false", help="Don't halt the test runner irrespective of the number of tests run") debugging_group.add_argument('--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected", default=True, action="store_false", help="Don't restart on an unexpected result") debugging_group.add_argument("--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") debugging_group.add_argument("--pdb", action="store_true", help="Drop into pdb on python exception") config_group = parser.add_argument_group("Configuration") config_group.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") config_group.add_argument('--binary-arg', default=[], action="append", dest="binary_args", help="Extra argument for the binary") config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use") config_group.add_argument('--webdriver-arg', default=[], action="append", dest="webdriver_args", help="Extra argument for the WebDriver binary") config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to root directory containing test metadata"), config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to root directory containing test files"), config_group.add_argument("--manifest", action="store", type=abs_path, dest="manifest_path", help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)") config_group.add_argument("--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info") config_group.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser against which to run tests") config_group.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") config_group.add_argument("--install-fonts", action="store_true", default=None, help="Allow the wptrunner to install fonts on your system") config_group.add_argument("--font-dir", action="store", type=abs_path, dest="font_dir", help="Path to local font installation directory", default=None) build_type = parser.add_mutually_exclusive_group() build_type.add_argument("--debug-build", dest="debug", action="store_true", default=None, help="Build is a debug build (overrides any mozinfo file)") build_type.add_argument("--release-build", dest="debug", action="store_false", default=None, help="Build is a release (overrides any mozinfo file)") chunking_group = parser.add_argument_group("Test Chunking") chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"], default=None, help="Chunking type to use") ssl_group = parser.add_argument_group("SSL/TLS") ssl_group.add_argument("--ssl-type", action="store", default=None, choices=["openssl", "pregenerated", "none"], help="Type of ssl support to enable (running without ssl may lead to spurious errors)") ssl_group.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl") ssl_group.add_argument("--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl") ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path, help="Path to ca certificate when using pregenerated ssl certificates") ssl_group.add_argument("--host-key-path", action="store", type=abs_path, help="Path to host private key when using pregenerated ssl certificates") ssl_group.add_argument("--host-cert-path", action="store", type=abs_path, help="Path to host certificate when using pregenerated ssl certificates") gecko_group = parser.add_argument_group("Gecko-specific") gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs") gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True, help="Run tests without electrolysis preferences") gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store", help="Path to directory containing assertion stack fixing scripts") gecko_group.add_argument("--setpref", dest="extra_prefs", action='append', default=[], metavar="PREF=VALUE", help="Defines an extra user preference (overrides those in prefs_root)") gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true", help="Enable leak checking") gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1, help="Number of parallel threads to use for stylo") gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true", default=None, help="Enable reftest runner implemented inside Marionette") gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false", help="Disable reftest runner implemented inside Marionette") gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store", choices=["always", "fail", "unexpected"], default="unexpected", help="With --reftest-internal, when to take a screenshot") gecko_group.add_argument("--chaos", dest="chaos_mode_flags", action="store", nargs="?", const=0xFFFFFFFF, type=int, help="Enable chaos mode with the specified feature flag " "(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for " "details). If no value is supplied, all features are activated") servo_group = parser.add_argument_group("Servo-specific") servo_group.add_argument("--user-stylesheet", default=[], action="append", dest="user_stylesheets", help="Inject a user CSS stylesheet into every test.") sauce_group = parser.add_argument_group("Sauce Labs-specific") sauce_group.add_argument("--sauce-browser", dest="sauce_browser", help="Sauce Labs browser name") sauce_group.add_argument("--sauce-platform", dest="sauce_platform", help="Sauce Labs OS platform") sauce_group.add_argument("--sauce-version", dest="sauce_version", help="Sauce Labs browser version") sauce_group.add_argument("--sauce-build", dest="sauce_build", help="Sauce Labs build identifier") sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*", help="Sauce Labs identifying tag", default=[]) sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id", help="Sauce Connect tunnel identifier") sauce_group.add_argument("--sauce-user", dest="sauce_user", help="Sauce Labs user name") sauce_group.add_argument("--sauce-key", dest="sauce_key", default=os.environ.get("SAUCE_ACCESS_KEY"), help="Sauce Labs access key") sauce_group.add_argument("--sauce-connect-binary", dest="sauce_connect_binary", help="Path to Sauce Connect binary") webkit_group = parser.add_argument_group("WebKit-specific") webkit_group.add_argument("--webkit-port", dest="webkit_port", help="WebKit port") parser.add_argument("test_list", nargs="*", help="List of URLs for tests to run, or paths including tests to run. " "(equivalent to --include)") commandline.log_formatters["wptreport"] = (formatters.WptreportFormatter, "wptreport format") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.") parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata"), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to web-platform-tests"), parser.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs"), parser.add_argument("--config", action="store", type=abs_path, help="Path to config file") parser.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") parser.add_argument("--test-types", action="store", nargs="*", default=["testharness", "reftest"], choices=["testharness", "reftest"], help="Test types to run") parser.add_argument("--processes", action="store", type=int, default=1, help="Number of simultaneous processes to use") parser.add_argument("--include", action="append", type=slash_prefixed, help="URL prefix to include") parser.add_argument("--exclude", action="append", type=slash_prefixed, help="URL prefix to exclude") parser.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") parser.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") parser.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"], default=None, help="Chunking type to use") parser.add_argument("--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") parser.add_argument("--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") parser.add_argument("--product", action="store", choices=product_choices, default="firefox", help="Browser against which to run tests") parser.add_argument('--debugger', help="run under a debugger, e.g. gdb or valgrind") parser.add_argument('--debugger-args', help="arguments to the debugger") parser.add_argument('--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") parser.add_argument("--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") parser.add_argument("--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") parser.add_argument("--b2g-no-backup", action="store_true", default=False, help="Don't backup device before testrun with --product=b2g") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser( description="""Runner for web-platform-tests tests.""", usage="""%(prog)s [OPTION]... [TEST]... TEST is either the full path to a test file to run, or the URL of a test excluding scheme host and port.""") parser.add_argument("--manifest-update", action="store_true", default=None, help="Regenerate the test manifest.") parser.add_argument("--no-manifest-update", action="store_false", dest="manifest_update", help="Prevent regeneration of the test manifest.") parser.add_argument( "--manifest-download", action="store_true", default=None, help="Attempt to download a preexisting manifest when updating.") parser.add_argument( "--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument( "--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--processes", action="store", type=int, default=None, help="Number of simultaneous processes to use") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") parser.add_argument( "--no-fail-on-unexpected", action="store_false", default=True, dest="fail_on_unexpected", help="Exit with status code 0 when test expectations are violated") mode_group = parser.add_argument_group("Mode") mode_group.add_argument( "--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") mode_group.add_argument( "--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") mode_group.add_argument("--list-tests", action="store_true", default=False, help="List all tests that will run") stability_group = mode_group.add_mutually_exclusive_group() stability_group.add_argument( "--verify", action="store_true", default=False, help="Run a stability check on the selected tests") stability_group.add_argument("--stability", action="store_true", default=False, help=argparse.SUPPRESS) mode_group.add_argument( "--verify-log-full", action="store_true", default=False, help="Output per-iteration test results when running verify") mode_group.add_argument( "--verify-repeat-loop", action="store", default=10, help= "Number of iterations for a run that reloads each test without restart.", type=int) mode_group.add_argument( "--verify-repeat-restart", action="store", default=5, help= "Number of iterations, for a run that restarts the runner between each iteration", type=int) chaos_mode_group = mode_group.add_mutually_exclusive_group() chaos_mode_group.add_argument( "--verify-no-chaos-mode", action="store_false", default=True, dest="verify_chaos_mode", help="Disable chaos mode when running on Firefox") chaos_mode_group.add_argument( "--verify-chaos-mode", action="store_true", default=True, dest="verify_chaos_mode", help="Enable chaos mode when running on Firefox") mode_group.add_argument( "--verify-max-time", action="store", default=None, help="The maximum number of minutes for the job to run", type=lambda x: timedelta(minutes=float(x))) output_results_group = mode_group.add_mutually_exclusive_group() output_results_group.add_argument( "--verify-no-output-results", action="store_false", dest="verify_output_results", default=True, help="Prints individuals test results and messages") output_results_group.add_argument( "--verify-output-results", action="store_true", dest="verify_output_results", default=True, help="Disable printing individuals test results and messages") test_selection_group = parser.add_argument_group("Test Selection") test_selection_group.add_argument("--test-types", action="store", nargs="*", default=wpttest.enabled_tests, choices=wpttest.enabled_tests, help="Test types to run") test_selection_group.add_argument("--include", action="append", help="URL prefix to include") test_selection_group.add_argument("--exclude", action="append", help="URL prefix to exclude") test_selection_group.add_argument( "--include-manifest", type=abs_path, help="Path to manifest listing tests to include") test_selection_group.add_argument( "--skip-timeout", action="store_true", help="Skip tests that are expected to time out") test_selection_group.add_argument( "--tag", action="append", dest="tags", help="Labels applied to tests to include in the run. " "Labels starting dir: are equivalent to top-level directories.") debugging_group = parser.add_argument_group("Debugging") debugging_group.add_argument( '--debugger', const="__default__", nargs="?", help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") debugging_group.add_argument( "--rerun", action="store", type=int, default=1, help="Number of times to re run each test without restarts") debugging_group.add_argument( "--repeat", action="store", type=int, default=1, help="Number of times to run the tests, restarting between each run") debugging_group.add_argument( "--repeat-until-unexpected", action="store_true", default=None, help="Run tests in a loop until one returns an unexpected result") debugging_group.add_argument( '--pause-after-test', action="store_true", default=None, help= "Halt the test runner after each test (this happens by default if only a single test is run)" ) debugging_group.add_argument( '--no-pause-after-test', dest="pause_after_test", action="store_false", help= "Don't halt the test runner irrespective of the number of tests run") debugging_group.add_argument( '--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected", default=True, action="store_false", help="Don't restart on an unexpected result") debugging_group.add_argument( "--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") debugging_group.add_argument( "--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") debugging_group.add_argument("--pdb", action="store_true", help="Drop into pdb on python exception") config_group = parser.add_argument_group("Configuration") config_group.add_argument("--binary", action="store", type=abs_path, help="Desktop binary to run tests against") config_group.add_argument('--binary-arg', default=[], action="append", dest="binary_args", help="Extra argument for the binary") config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use") config_group.add_argument('--webdriver-arg', default=[], action="append", dest="webdriver_args", help="Extra argument for the WebDriver binary") config_group.add_argument("--package-name", action="store", help="Android package name to run tests against") config_group.add_argument( "--device-serial", action="store", help="Running Android instance to connect to, if not emulator-5554") config_group.add_argument( "--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to root directory containing test metadata"), config_group.add_argument( "--tests", action="store", type=abs_path, dest="tests_root", help="Path to root directory containing test files"), config_group.add_argument( "--manifest", action="store", type=abs_path, dest="manifest_path", help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)" ) config_group.add_argument( "--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info" ) config_group.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser against which to run tests") config_group.add_argument( "--browser-version", action="store", default=None, help="Informative string detailing the browser " "release version. This is included in the run_info data.") config_group.add_argument( "--browser-channel", action="store", default=None, help="Informative string detailing the browser " "release channel. This is included in the run_info data.") config_group.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") config_group.add_argument( "--install-fonts", action="store_true", default=None, help="Allow the wptrunner to install fonts on your system") config_group.add_argument("--font-dir", action="store", type=abs_path, dest="font_dir", help="Path to local font installation directory", default=None) config_group.add_argument("--headless", action="store_true", help="Run browser in headless mode", default=None) config_group.add_argument("--no-headless", action="store_false", dest="headless", help="Don't run browser in headless mode") build_type = parser.add_mutually_exclusive_group() build_type.add_argument( "--debug-build", dest="debug", action="store_true", default=None, help="Build is a debug build (overrides any mozinfo file)") build_type.add_argument( "--release-build", dest="debug", action="store_false", default=None, help="Build is a release (overrides any mozinfo file)") chunking_group = parser.add_argument_group("Test Chunking") chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") chunking_group.add_argument( "--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"], default=None, help="Chunking type to use") ssl_group = parser.add_argument_group("SSL/TLS") ssl_group.add_argument( "--ssl-type", action="store", default=None, choices=["openssl", "pregenerated", "none"], help= "Type of ssl support to enable (running without ssl may lead to spurious errors)" ) ssl_group.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl") ssl_group.add_argument( "--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl") ssl_group.add_argument( "--ca-cert-path", action="store", type=abs_path, help="Path to ca certificate when using pregenerated ssl certificates") ssl_group.add_argument( "--host-key-path", action="store", type=abs_path, help="Path to host private key when using pregenerated ssl certificates" ) ssl_group.add_argument( "--host-cert-path", action="store", type=abs_path, help="Path to host certificate when using pregenerated ssl certificates" ) gecko_group = parser.add_argument_group("Gecko-specific") gecko_group.add_argument( "--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs") gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True, help="Run tests without electrolysis preferences") gecko_group.add_argument( "--stackfix-dir", dest="stackfix_dir", action="store", help="Path to directory containing assertion stack fixing scripts") gecko_group.add_argument( "--setpref", dest="extra_prefs", action='append', default=[], metavar="PREF=VALUE", help="Defines an extra user preference (overrides those in prefs_root)" ) gecko_group.add_argument( "--leak-check", dest="leak_check", action="store_true", default=None, help="Enable leak checking (enabled by default for debug builds, " "silently ignored for opt)") gecko_group.add_argument("--no-leak-check", dest="leak_check", action="store_false", default=None, help="Disable leak checking") gecko_group.add_argument( "--stylo-threads", action="store", type=int, default=1, help="Number of parallel threads to use for stylo") gecko_group.add_argument( "--reftest-internal", dest="reftest_internal", action="store_true", default=None, help="Enable reftest runner implemented inside Marionette") gecko_group.add_argument( "--reftest-external", dest="reftest_internal", action="store_false", help="Disable reftest runner implemented inside Marionette") gecko_group.add_argument( "--reftest-screenshot", dest="reftest_screenshot", action="store", choices=["always", "fail", "unexpected"], default="unexpected", help="With --reftest-internal, when to take a screenshot") gecko_group.add_argument( "--chaos", dest="chaos_mode_flags", action="store", nargs="?", const=0xFFFFFFFF, type=int, help="Enable chaos mode with the specified feature flag " "(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for " "details). If no value is supplied, all features are activated") servo_group = parser.add_argument_group("Servo-specific") servo_group.add_argument( "--user-stylesheet", default=[], action="append", dest="user_stylesheets", help="Inject a user CSS stylesheet into every test.") sauce_group = parser.add_argument_group("Sauce Labs-specific") sauce_group.add_argument("--sauce-browser", dest="sauce_browser", help="Sauce Labs browser name") sauce_group.add_argument("--sauce-platform", dest="sauce_platform", help="Sauce Labs OS platform") sauce_group.add_argument("--sauce-version", dest="sauce_version", help="Sauce Labs browser version") sauce_group.add_argument("--sauce-build", dest="sauce_build", help="Sauce Labs build identifier") sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*", help="Sauce Labs identifying tag", default=[]) sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id", help="Sauce Connect tunnel identifier") sauce_group.add_argument("--sauce-user", dest="sauce_user", help="Sauce Labs user name") sauce_group.add_argument("--sauce-key", dest="sauce_key", default=os.environ.get("SAUCE_ACCESS_KEY"), help="Sauce Labs access key") sauce_group.add_argument("--sauce-connect-binary", dest="sauce_connect_binary", help="Path to Sauce Connect binary") sauce_group.add_argument("--sauce-init-timeout", action="store", type=int, default=30, help="Number of seconds to wait for Sauce " "Connect tunnel to be available before " "aborting") sauce_group.add_argument("--sauce-connect-arg", action="append", default=[], dest="sauce_connect_args", help="Command-line argument to forward to the " "Sauce Connect binary (repeatable)") webkit_group = parser.add_argument_group("WebKit-specific") webkit_group.add_argument("--webkit-port", dest="webkit_port", help="WebKit port") parser.add_argument( "test_list", nargs="*", help="List of URLs for tests to run, or paths including tests to run. " "(equivalent to --include)") commandline.log_formatters["wptreport"] = (formatters.WptreportFormatter, "wptreport format") commandline.add_logging_group(parser) return parser
def create_parser_update(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser( "web-platform-tests-update", description="Update script for web-platform-tests tests.") parser.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser for which metadata is being updated") parser.add_argument("--config", action="store", type=abs_path, help="Path to config file") parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata"), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to web-platform-tests"), parser.add_argument( "--sync-path", action="store", type=abs_path, help="Path to store git checkout of web-platform-tests during update"), parser.add_argument( "--remote_url", action="store", help="URL of web-platfrom-tests repository to sync against"), parser.add_argument("--branch", action="store", type=abs_path, help="Remote branch to sync against") parser.add_argument("--rev", action="store", help="Revision to sync to") parser.add_argument( "--no-patch", action="store_true", help="Don't create an mq patch or git commit containing the changes.") parser.add_argument("--sync", dest="sync", action="store_true", default=False, help="Sync the tests with the latest from upstream") parser.add_argument( "--ignore-existing", action="store_true", help= "When updating test results only consider results from the logfiles provided, not existing expectations." ) parser.add_argument( "--continue", action="store_true", help="Continue a previously started run of the update script") parser.add_argument( "--abort", action="store_true", help="Clear state from a previous incomplete run of the update script") # Should make this required iff run=logfile parser.add_argument("run_log", nargs="*", type=abs_path, help="Log file from run of tests") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser( description="""Runner for web-platform-tests tests.""", usage="""%(prog)s [OPTION]... [TEST]... TEST is either the full path to a test file to run, or the URL of a test excluding scheme host and port.""") parser.add_argument("--manifest-update", action="store_true", default=False, help="Regenerate the test manifest.") parser.add_argument( "--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument( "--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--processes", action="store", type=int, default=None, help="Number of simultaneous processes to use") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") mode_group = parser.add_argument_group("Mode") mode_group.add_argument( "--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") mode_group.add_argument( "--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") test_selection_group = parser.add_argument_group("Test Selection") test_selection_group.add_argument("--test-types", action="store", nargs="*", default=wpttest.enabled_tests, choices=wpttest.enabled_tests, help="Test types to run") test_selection_group.add_argument("--include", action="append", help="URL prefix to include") test_selection_group.add_argument("--exclude", action="append", help="URL prefix to exclude") test_selection_group.add_argument( "--include-manifest", type=abs_path, help="Path to manifest listing tests to include") test_selection_group.add_argument( "--tag", action="append", dest="tags", help= "Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories." ) debugging_group = parser.add_argument_group("Debugging") debugging_group.add_argument( '--debugger', const="__default__", nargs="?", help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") debugging_group.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") debugging_group.add_argument( "--repeat-until-unexpected", action="store_true", default=None, help="Run tests in a loop until one returns an unexpected result") debugging_group.add_argument( '--pause-after-test', action="store_true", default=None, help= "Halt the test runner after each test (this happens by default if only a single test is run)" ) debugging_group.add_argument( '--no-pause-after-test', dest="pause_after_test", action="store_false", help= "Don't halt the test runner irrespective of the number of tests run") debugging_group.add_argument( '--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") debugging_group.add_argument( "--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") debugging_group.add_argument( "--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") debugging_group.add_argument("--pdb", action="store_true", help="Drop into pdb on python exception") config_group = parser.add_argument_group("Configuration") config_group.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") config_group.add_argument('--binary-arg', default=[], action="append", dest="binary_args", help="Extra argument for the binary (servo)") config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use") config_group.add_argument( "--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to root directory containing test metadata"), config_group.add_argument( "--tests", action="store", type=abs_path, dest="tests_root", help="Path to root directory containing test files"), config_group.add_argument( "--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info" ) config_group.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser against which to run tests") config_group.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") build_type = parser.add_mutually_exclusive_group() build_type.add_argument( "--debug-build", dest="debug", action="store_true", default=None, help="Build is a debug build (overrides any mozinfo file)") build_type.add_argument( "--release-build", dest="debug", action="store_false", default=None, help="Build is a release (overrides any mozinfo file)") chunking_group = parser.add_argument_group("Test Chunking") chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") chunking_group.add_argument( "--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"], default=None, help="Chunking type to use") ssl_group = parser.add_argument_group("SSL/TLS") ssl_group.add_argument( "--ssl-type", action="store", default=None, choices=["openssl", "pregenerated", "none"], help= "Type of ssl support to enable (running without ssl may lead to spurious errors)" ) ssl_group.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl") ssl_group.add_argument( "--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl") ssl_group.add_argument( "--ca-cert-path", action="store", type=abs_path, help="Path to ca certificate when using pregenerated ssl certificates") ssl_group.add_argument( "--host-key-path", action="store", type=abs_path, help="Path to host private key when using pregenerated ssl certificates" ) ssl_group.add_argument( "--host-cert-path", action="store", type=abs_path, help="Path to host certificate when using pregenerated ssl certificates" ) gecko_group = parser.add_argument_group("Gecko-specific") gecko_group.add_argument( "--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs") gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True, help="Run tests without electrolysis preferences") b2g_group = parser.add_argument_group("B2G-specific") b2g_group.add_argument( "--b2g-no-backup", action="store_true", default=False, help="Don't backup device before testrun with --product=b2g") servo_group = parser.add_argument_group("Servo-specific") servo_group.add_argument( "--user-stylesheet", default=[], action="append", dest="user_stylesheets", help="Inject a user CSS stylesheet into every test.") servo_group.add_argument("--servo-backend", default="cpu", choices=["cpu", "webrender"], help="Rendering backend to use with Servo.") parser.add_argument( "test_list", nargs="*", help="List of URLs for tests to run, or paths including tests to run. " "(equivalent to --include)") commandline.add_logging_group(parser) return parser
def create_parser_update(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser( "web-platform-tests-update", description="Update script for web-platform-tests tests.") parser.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser for which metadata is being updated") parser.add_argument("--config", action="store", type=abs_path, help="Path to config file") parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata"), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to web-platform-tests"), parser.add_argument( "--manifest", action="store", type=abs_path, dest="manifest_path", help="Path to test manifest (default is ${metadata_root}/MANIFEST.json)" ) parser.add_argument( "--sync-path", action="store", type=abs_path, help="Path to store git checkout of web-platform-tests during update"), parser.add_argument( "--remote_url", action="store", help="URL of web-platfrom-tests repository to sync against"), parser.add_argument("--branch", action="store", type=abs_path, help="Remote branch to sync against") parser.add_argument("--rev", action="store", help="Revision to sync to") parser.add_argument("--patch", action="store_true", dest="patch", default=None, help="Create a VCS commit containing the changes.") parser.add_argument( "--no-patch", action="store_false", dest="patch", help="Don't create a VCS commit containing the changes.") parser.add_argument( "--sync", dest="sync", action="store_true", default=False, help="Sync the tests with the latest from upstream (implies --patch)") parser.add_argument( "--ignore-existing", action="store_true", help= "When updating test results only consider results from the logfiles provided, not existing expectations." ) parser.add_argument( "--stability", nargs="?", action="store", const="unstable", default=None, help= ("Reason for disabling tests. When updating test results, disable tests that have " "inconsistent results across many runs with the given reason.")) parser.add_argument( "--no-remove-obsolete", action="store_false", dest="remove_obsolete", default=True, help= ("Don't remove metadata files that no longer correspond to a test file" )) parser.add_argument( "--no-store-state", action="store_false", dest="store_state", help="Store state so that steps can be resumed after failure") parser.add_argument( "--continue", action="store_true", help="Continue a previously started run of the update script") parser.add_argument( "--abort", action="store_true", help="Clear state from a previous incomplete run of the update script") parser.add_argument( "--exclude", action="store", nargs="*", help="List of glob-style paths to exclude when syncing tests") parser.add_argument( "--include", action="store", nargs="*", help= "List of glob-style paths to include which would otherwise be excluded when syncing tests" ) parser.add_argument( "--extra-property", action="append", default=[], help="Extra property from run_info.json to use in metadata update") # Should make this required iff run=logfile parser.add_argument("run_log", nargs="*", type=abs_path, help="Log file from run of tests") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.") parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata"), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to test files"), parser.add_argument("--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info") parser.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") parser.add_argument("--manifest-update", action="store_true", default=False, help="Force regeneration of the test manifest") parser.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") parser.add_argument("--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use") parser.add_argument("--processes", action="store", type=int, default=None, help="Number of simultaneous processes to use") parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") parser.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser against which to run tests") parser.add_argument("--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") parser.add_argument("--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") test_selection_group = parser.add_argument_group("Test Selection") test_selection_group.add_argument("--test-types", action="store", nargs="*", default=["testharness", "reftest"], choices=["testharness", "reftest"], help="Test types to run") test_selection_group.add_argument("--include", action="append", help="URL prefix to include") test_selection_group.add_argument("--exclude", action="append", help="URL prefix to exclude") test_selection_group.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") debugging_group = parser.add_argument_group("Debugging") debugging_group.add_argument('--debugger', const="__default__", nargs="?", help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") debugging_group.add_argument('--pause-after-test', action="store_true", default=None, help="Halt the test runner after each test (this happens by default if only a single test is run)") debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false", help="Don't halt the test runner irrespective of the number of tests run") debugging_group.add_argument('--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") debugging_group.add_argument("--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") chunking_group = parser.add_argument_group("Test Chunking") chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"], default=None, help="Chunking type to use") ssl_group = parser.add_argument_group("SSL/TLS") ssl_group.add_argument("--ssl-type", action="store", default=None, choices=["openssl", "pregenerated", "none"], help="Type of ssl support to enable (running without ssl may lead to spurious errors)") ssl_group.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl") ssl_group.add_argument("--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl") ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path, help="Path to ca certificate when using pregenerated ssl certificates") ssl_group.add_argument("--host-key-path", action="store", type=abs_path, help="Path to host private key when using pregenerated ssl certificates") ssl_group.add_argument("--host-cert-path", action="store", type=abs_path, help="Path to host certificate when using pregenerated ssl certificates") gecko_group = parser.add_argument_group("Gecko-specific") gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs") b2g_group = parser.add_argument_group("B2G-specific") b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False, help="Don't backup device before testrun with --product=b2g") parser.add_argument("test_list", nargs="*", help="List of URLs for tests to run, or paths including tests to run. " "(equivalent to --include)") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser(description="""Runner for web-platform-tests tests.""", usage="""%(prog)s [OPTION]... [TEST]... TEST is either the full path to a test file to run, or the URL of a test excluding scheme host and port.""") parser.add_argument("--manifest-update", action="store_true", default=False, help="Regenerate the test manifest.") parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--processes", action="store", type=int, default=None, help="Number of simultaneous processes to use") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") mode_group = parser.add_argument_group("Mode") mode_group.add_argument("--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") mode_group.add_argument("--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") test_selection_group = parser.add_argument_group("Test Selection") test_selection_group.add_argument("--test-types", action="store", nargs="*", default=wpttest.enabled_tests, choices=wpttest.enabled_tests, help="Test types to run") test_selection_group.add_argument("--include", action="append", help="URL prefix to include") test_selection_group.add_argument("--exclude", action="append", help="URL prefix to exclude") test_selection_group.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") test_selection_group.add_argument("--tag", action="append", dest="tags", help="Labels applied to tests to include in the run. Labels starting dir: are equivalent to top-level directories.") debugging_group = parser.add_argument_group("Debugging") debugging_group.add_argument('--debugger', const="__default__", nargs="?", help="run under a debugger, e.g. gdb or valgrind") debugging_group.add_argument('--debugger-args', help="arguments to the debugger") debugging_group.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") debugging_group.add_argument("--repeat-until-unexpected", action="store_true", default=None, help="Run tests in a loop until one returns an unexpected result") debugging_group.add_argument('--pause-after-test', action="store_true", default=None, help="Halt the test runner after each test (this happens by default if only a single test is run)") debugging_group.add_argument('--no-pause-after-test', dest="pause_after_test", action="store_false", help="Don't halt the test runner irrespective of the number of tests run") debugging_group.add_argument('--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") debugging_group.add_argument('--no-restart-on-unexpected', dest="restart_on_unexpected", default=True, action="store_false", help="Don't restart on an unexpected result") debugging_group.add_argument("--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") debugging_group.add_argument("--pdb", action="store_true", help="Drop into pdb on python exception") config_group = parser.add_argument_group("Configuration") config_group.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") config_group.add_argument('--binary-arg', default=[], action="append", dest="binary_args", help="Extra argument for the binary") config_group.add_argument("--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use") config_group.add_argument('--webdriver-arg', default=[], action="append", dest="webdriver_args", help="Extra argument for the WebDriver binary") config_group.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to root directory containing test metadata"), config_group.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to root directory containing test files"), config_group.add_argument("--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info") config_group.add_argument("--product", action="store", choices=product_choices, default=None, help="Browser against which to run tests") config_group.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") build_type = parser.add_mutually_exclusive_group() build_type.add_argument("--debug-build", dest="debug", action="store_true", default=None, help="Build is a debug build (overrides any mozinfo file)") build_type.add_argument("--release-build", dest="debug", action="store_false", default=None, help="Build is a release (overrides any mozinfo file)") chunking_group = parser.add_argument_group("Test Chunking") chunking_group.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") chunking_group.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash", "dir_hash"], default=None, help="Chunking type to use") ssl_group = parser.add_argument_group("SSL/TLS") ssl_group.add_argument("--ssl-type", action="store", default=None, choices=["openssl", "pregenerated", "none"], help="Type of ssl support to enable (running without ssl may lead to spurious errors)") ssl_group.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl") ssl_group.add_argument("--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl") ssl_group.add_argument("--ca-cert-path", action="store", type=abs_path, help="Path to ca certificate when using pregenerated ssl certificates") ssl_group.add_argument("--host-key-path", action="store", type=abs_path, help="Path to host private key when using pregenerated ssl certificates") ssl_group.add_argument("--host-cert-path", action="store", type=abs_path, help="Path to host certificate when using pregenerated ssl certificates") gecko_group = parser.add_argument_group("Gecko-specific") gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs") gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True, help="Run tests without electrolysis preferences") gecko_group.add_argument("--stackfix-dir", dest="stackfix_dir", action="store", help="Path to directory containing assertion stack fixing scripts") gecko_group.add_argument("--setpref", dest="extra_prefs", action='append', default=[], metavar="PREF=VALUE", help="Defines an extra user preference (overrides those in prefs_root)") gecko_group.add_argument("--leak-check", dest="leak_check", action="store_true", help="Enable leak checking") servo_group = parser.add_argument_group("Servo-specific") servo_group.add_argument("--user-stylesheet", default=[], action="append", dest="user_stylesheets", help="Inject a user CSS stylesheet into every test.") sauce_group = parser.add_argument_group("Sauce Labs-specific") sauce_group.add_argument("--sauce-browser", dest="sauce_browser", help="Sauce Labs browser name") sauce_group.add_argument("--sauce-platform", dest="sauce_platform", help="Sauce Labs OS platform") sauce_group.add_argument("--sauce-version", dest="sauce_version", help="Sauce Labs browser version") sauce_group.add_argument("--sauce-build", dest="sauce_build", help="Sauce Labs build identifier") sauce_group.add_argument("--sauce-tags", dest="sauce_tags", nargs="*", help="Sauce Labs identifying tag", default=[]) sauce_group.add_argument("--sauce-tunnel-id", dest="sauce_tunnel_id", help="Sauce Connect tunnel identifier") sauce_group.add_argument("--sauce-user", dest="sauce_user", help="Sauce Labs user name") sauce_group.add_argument("--sauce-key", dest="sauce_key", default=os.environ.get("SAUCE_ACCESS_KEY"), help="Sauce Labs access key") sauce_group.add_argument("--sauce-connect-binary", dest="sauce_connect_binary", help="Path to Sauce Connect binary") parser.add_argument("test_list", nargs="*", help="List of URLs for tests to run, or paths including tests to run. " "(equivalent to --include)") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.") parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata"), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to test files"), parser.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs"), parser.add_argument("--serve-root", action="store", type=abs_path, dest="serve_root", help="Path to web-platform-tests checkout containing serve.py and manifest.py" " (defaults to test_root)") parser.add_argument("--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info") parser.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") parser.add_argument("--manifest-update", action="store_true", default=False, help="Force regeneration of the test manifest") parser.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") parser.add_argument("--test-types", action="store", nargs="*", default=["testharness", "reftest"], choices=["testharness", "reftest"], help="Test types to run") parser.add_argument("--processes", action="store", type=int, default=1, help="Number of simultaneous processes to use") parser.add_argument("--include", action="append", type=slash_prefixed, help="URL prefix to include") parser.add_argument("--exclude", action="append", type=slash_prefixed, help="URL prefix to exclude") parser.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") parser.add_argument("--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory") parser.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") parser.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") parser.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"], default=None, help="Chunking type to use") parser.add_argument("--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.") parser.add_argument("--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform") parser.add_argument("--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use") parser.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") parser.add_argument("--product", action="store", choices=product_choices, default="firefox", help="Browser against which to run tests") parser.add_argument('--debugger', help="run under a debugger, e.g. gdb or valgrind") parser.add_argument('--debugger-args', help="arguments to the debugger") parser.add_argument('--pause-on-unexpected', action="store_true", help="Halt the test runner when an unexpected result is encountered") parser.add_argument("--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.") parser.add_argument("--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.") parser.add_argument("--b2g-no-backup", action="store_true", default=False, help="Don't backup device before testrun with --product=b2g") commandline.add_logging_group(parser) return parser
def create_parser(product_choices=None): from mozlog.structured import commandline import products if product_choices is None: config_data = config.load() product_choices = products.products_enabled(config_data) parser = argparse.ArgumentParser(description="Runner for web-platform-tests tests.") parser.add_argument( "--metadata", action="store", type=abs_path, dest="metadata_root", help="Path to the folder containing test metadata", ), parser.add_argument("--tests", action="store", type=abs_path, dest="tests_root", help="Path to test files"), parser.add_argument( "--prefs-root", dest="prefs_root", action="store", type=abs_path, help="Path to the folder containing browser prefs", ), parser.add_argument( "--serve-root", action="store", type=abs_path, dest="serve_root", help="Path to web-platform-tests checkout containing serve.py and manifest.py" " (defaults to test_root)", ) parser.add_argument( "--run-info", action="store", type=abs_path, help="Path to directory containing extra json files to add to run info", ) parser.add_argument("--config", action="store", type=abs_path, dest="config", help="Path to config file") parser.add_argument( "--manifest-update", action="store_true", default=False, help="Force regeneration of the test manifest" ) parser.add_argument("--binary", action="store", type=abs_path, help="Binary to run tests against") parser.add_argument( "--webdriver-binary", action="store", metavar="BINARY", type=abs_path, help="WebDriver server binary to use" ) parser.add_argument( "--test-types", action="store", nargs="*", default=["testharness", "reftest"], choices=["testharness", "reftest"], help="Test types to run", ) parser.add_argument( "--processes", action="store", type=int, default=1, help="Number of simultaneous processes to use" ) parser.add_argument("--include", action="append", type=slash_prefixed, help="URL prefix to include") parser.add_argument("--exclude", action="append", type=slash_prefixed, help="URL prefix to exclude") parser.add_argument("--include-manifest", type=abs_path, help="Path to manifest listing tests to include") parser.add_argument( "--run-by-dir", type=int, nargs="?", default=False, help="Split run into groups by directories. With a parameter," "limit the depth of splits e.g. --run-by-dir=1 to split by top-level" "directory", ) parser.add_argument("--total-chunks", action="store", type=int, default=1, help="Total number of chunks to use") parser.add_argument("--this-chunk", action="store", type=int, default=1, help="Chunk number to run") parser.add_argument( "--chunk-type", action="store", choices=["none", "equal_time", "hash"], default=None, help="Chunking type to use", ) parser.add_argument( "--list-test-groups", action="store_true", default=False, help="List the top level directories containing tests that will run.", ) parser.add_argument( "--list-disabled", action="store_true", default=False, help="List the tests that are disabled on the current platform", ) parser.add_argument( "--timeout-multiplier", action="store", type=float, default=None, help="Multiplier relative to standard test timeout to use", ) parser.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") parser.add_argument( "--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging" ) parser.add_argument( "--product", action="store", choices=product_choices, default="firefox", help="Browser against which to run tests", ) parser.add_argument("--debugger", help="run under a debugger, e.g. gdb or valgrind") parser.add_argument("--debugger-args", help="arguments to the debugger") parser.add_argument( "--pause-on-unexpected", action="store_true", help="Halt the test runner when an unexpected result is encountered", ) parser.add_argument( "--symbols-path", action="store", type=url_or_path, help="Path or url to symbols file used to analyse crash minidumps.", ) parser.add_argument( "--stackwalk-binary", action="store", type=abs_path, help="Path to stackwalker program used to analyse minidumps.", ) parser.add_argument( "--ssl-type", action="store", default=None, choices=["openssl", "pregenerated", "none"], help="Type of ssl support to enable (running without ssl may lead to spurious errors)", ) parser.add_argument("--openssl-binary", action="store", help="Path to openssl binary", default="openssl") parser.add_argument("--certutil-binary", action="store", help="Path to certutil binary for use with Firefox + ssl") parser.add_argument( "--ca-cert-path", action="store", type=abs_path, help="Path to ca certificate when using pregenerated ssl certificates", ) parser.add_argument( "--host-key-path", action="store", type=abs_path, help="Path to host private key when using pregenerated ssl certificates", ) parser.add_argument( "--host-cert-path", action="store", type=abs_path, help="Path to host certificate when using pregenerated ssl certificates", ) parser.add_argument( "--b2g-no-backup", action="store_true", default=False, help="Don't backup device before testrun with --product=b2g", ) commandline.add_logging_group(parser) return parser