def _parse_args(self, args=None): print_options = printing.print_options() perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option( "--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option( "--build-directory", help= "Path to the directory under which build files are kept (should not include configuration)" ), optparse.make_option("--time-out-ms", default=30000, help="Set the timeout for each test"), ] option_list = (perf_option_list + print_options) return optparse.OptionParser(option_list=option_list).parse_args(args)
def _parse_args(args=None): print_options = printing.print_options() perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--builder-name", help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")), optparse.make_option("--build-number", help=("The build number of the builder running this script.")), optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date (default)."), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=600 * 1000, help="Set the timeout for each test"), optparse.make_option("--output-json-path", help="Filename of the JSON file that summaries the results"), optparse.make_option("--source-json-path", help="Path to a JSON file to be merged into the JSON file when --output-json-path is present"), optparse.make_option("--test-results-server", help="Upload the generated JSON file to the specified server when --output-json-path is present"), ] option_list = (perf_option_list + print_options) return optparse.OptionParser(option_list=option_list).parse_args(args)
def get_options(worker_model): option_list = (manager_worker_broker.runtime_options() + printing.print_options() + [optparse.make_option("--experimental-fully-parallel", default=False), optparse.make_option("--child-processes", default='2')]) parser = optparse.OptionParser(option_list=option_list) options, args = parser.parse_args(args=['--worker-model', worker_model]) return options
def _parse_args(args=None): print_options = printing.print_options() perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option( "--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option( "--builder-name", help= ("The name of the builder shown on the waterfall running this script e.g. google-mac-2." )), optparse.make_option( "--build-number", help=("The build number of the builder running this script.")), optparse.make_option( "--build", dest="build", action="store_true", default=True, help= "Check to ensure the DumpRenderTree build is up-to-date (default)." ), optparse.make_option( "--build-directory", help= "Path to the directory under which build files are kept (should not include configuration)" ), optparse.make_option("--time-out-ms", default=240 * 1000, help="Set the timeout for each test"), optparse.make_option( "--output-json-path", help="Filename of the JSON file that summaries the results"), optparse.make_option( "--source-json-path", help= "Path to a JSON file to be merged into the JSON file when --output-json-path is present" ), optparse.make_option( "--test-results-server", help= "Upload the generated JSON file to the specified server when --output-json-path is present" ), ] option_list = (perf_option_list + print_options) return optparse.OptionParser(option_list=option_list).parse_args(args)
def get_printer(self, args=None): args = args or [] printing_options = printing.print_options() option_parser = optparse.OptionParser(option_list=printing_options) options, args = option_parser.parse_args(args) host = MockHost() self._port = host.port_factory.get('test', options) regular_output = StringIO.StringIO() printer = printing.Printer(self._port, options, regular_output) return printer, regular_output
def get_printer(self, args=None, tty=False): args = args or [] printing_options = printing.print_options() option_parser = optparse.OptionParser(option_list=printing_options) options, args = option_parser.parse_args(args) self._port = port.get('test', options) nproc = 2 regular_output = array_stream.ArrayStream(tty=tty) buildbot_output = array_stream.ArrayStream() printer = printing.Printer(self._port, options, regular_output, buildbot_output, configure_logging=True) return printer, regular_output, buildbot_output
def get_printer(self, args=None, tty=False): args = args or [] printing_options = printing.print_options() option_parser = optparse.OptionParser(option_list=printing_options) options, args = option_parser.parse_args(args) host = MockHost() self._port = host.port_factory.get("test", options) nproc = 2 regular_output = StringIO.StringIO() regular_output.isatty = lambda: tty buildbot_output = StringIO.StringIO() printer = printing.Printer(self._port, options, regular_output, buildbot_output) return printer, regular_output, buildbot_output
def _parse_args(self, args=None): print_options = printing.print_options() perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), optparse.make_option("--platform", help="Specify port/platform being tested (i.e. chromium-mac)"), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--time-out-ms", default=30000, help="Set the timeout for each test"), ] option_list = (perf_option_list + print_options) return optparse.OptionParser(option_list=option_list).parse_args(args)
def get_options(args): print_options = printing.print_options() option_parser = optparse.OptionParser(option_list=print_options) return option_parser.parse_args(args)
def parse_args(args): option_group_definitions = [] option_group_definitions.append(("Platform options", platform_options())) option_group_definitions.append(("Configuration options", configuration_options())) option_group_definitions.append(("Printing Options", printing.print_options())) option_group_definitions.append(("EFL-specific Options", [ optparse.make_option("--webprocess-cmd-prefix", type="string", default=False, help="Prefix used when spawning the Web process (Debug mode only)"), ])) option_group_definitions.append(("WebKit Options", [ optparse.make_option("--gc-between-tests", action="store_true", default=False, help="Force garbage collection between each test"), optparse.make_option("--complex-text", action="store_true", default=False, help="Use the complex text code path for all text (Mac OS X and Windows only)"), optparse.make_option("-l", "--leaks", action="store_true", default=False, help="Enable leaks checking (Mac OS X only)"), optparse.make_option("-g", "--guard-malloc", action="store_true", default=False, help="Enable Guard Malloc (Mac OS X only)"), optparse.make_option("--threaded", action="store_true", default=False, help="Run a concurrent JavaScript thread with each test"), optparse.make_option("--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree."), # FIXME: We should merge this w/ --build-directory and only have one flag. optparse.make_option("--root", action="store", help="Path to a directory containing the executables needed to run tests."), ])) option_group_definitions.append(("Results Options", [ optparse.make_option("-p", "--pixel", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-pixel", "--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-sample-on-timeout", action="store_false", dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"), optparse.make_option("--no-ref-tests", action="store_true", dest="no_ref_tests", help="Skip all ref tests"), optparse.make_option("--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--add-platform-exceptions", action="store_true", default=False, help="Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory"), optparse.make_option("--new-baseline", action="store_true", default=False, help="Save generated results as new baselines " "into the *most-specific-platform* directory, overwriting whatever's " "already there. Equivalent to --reset-results --add-platform-exceptions"), optparse.make_option("--reset-results", action="store_true", default=False, help="Reset expectations to the " "generated results in their existing location."), optparse.make_option("--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist"), #FIXME: we should support a comma separated list with --pixel-test-directory as well. optparse.make_option("--pixel-test-directory", action="append", default=[], dest="pixel_test_directories", help="A directory where it is allowed to execute tests as pixel tests. " "Specify multiple times to add multiple directories. " "This option implies --pixel-tests. If specified, only those tests " "will be executed as pixel tests that are located in one of the " "directories enumerated with the option. Some ports may ignore this " "option while others can have a default value that can be overridden here."), optparse.make_option("--skip-failing-tests", action="store_true", default=False, help="Skip tests that are expected to fail. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option("--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option("--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option("--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option("--additional-expectations", action="append", default=[], help="Path to a test_expectations file that will override previous expectations. " "Specify multiple times for multiple sets of overrides."), optparse.make_option("--compare-port", action="store", default=None, help="Use the specified port's baselines first"), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), optparse.make_option("--full-results-html", action="store_true", default=False, help="Show all failures in results.html, rather than only regressions"), optparse.make_option("--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option("--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), optparse.make_option("--ignore-metrics", action="store_true", dest="ignore_metrics", default=False, help="Ignore rendering metrics related information from test " "output, only compare the structure of the rendertree."), optparse.make_option("--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), ])) option_group_definitions.append(("Testing Options", [ optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option("-n", "--dry-run", action="store_true", default=False, help="Do everything but actually run the tests or upload results."), optparse.make_option("--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), optparse.make_option("-i", "--ignore-tests", action="append", default=[], help="directories or test to ignore (may specify multiple times)"), optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), optparse.make_option("--skipped", action="store", default="default", help=("control how tests marked SKIP are run. " "'default' == Skip tests unless explicitly listed on the command line, " "'ignore' == Run them anyway, " "'only' == only run the SKIP tests, " "'always' == always skip, even if listed on the command line.")), optparse.make_option("--force", dest="skipped", action="store_const", const='ignore', help="Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)"), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), optparse.make_option("--order", action="store", default="natural", help=("determine the order in which the test cases will be run. " "'none' == use the order in which the tests were listed either in arguments or test list, " "'natural' == use the natural order (default), " "'random' == randomize the test order.")), optparse.make_option("--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option("--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), optparse.make_option("--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), optparse.make_option("--run-singly", action="store_true", default=False, help="run a separate DumpRenderTree for each test (implies --verbose)"), optparse.make_option("--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option("--exit-after-n-failures", type="int", default=None, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", default=None, help="Exit after the first N crashes instead of " "running all tests"), optparse.make_option("--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)"), optparse.make_option("--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"), optparse.make_option("--retry-failures", action="store_true", default=True, help="Re-try any tests that produce unexpected results (default)"), optparse.make_option("--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results."), optparse.make_option("--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards"), optparse.make_option("--additional-env-var", type="string", action="append", default=[], help="Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)"), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option("--profiler", action="store", help="Output per-test profile information, using the specified profiler."), ])) option_group_definitions.append(("Miscellaneous Options", [ optparse.make_option("--lint-test-files", action="store_true", default=False, help=("Makes sure the test files parse for all " "configurations. Does not run any tests.")), ])) # FIXME: Move these into json_results_generator.py option_group_definitions.append(("Result JSON Options", [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option("--builder-name", default="", help=("The name of the builder shown on the waterfall running " "this script e.g. WebKit.")), optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", help=("The name of the builder used in its path, e.g. " "webkit-rel.")), optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running this script.")), optparse.make_option("--test-results-server", default="", help=("If specified, upload results json files to this appengine " "server.")), ])) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)
def parse_args(args=None): """Provides a default set of command line args. Returns a tuple of options, args from optparse""" option_group_definitions = [] option_group_definitions.append(("Configuration options", port_options())) option_group_definitions.append(("Printing Options", printing.print_options())) # FIXME: These options should move onto the ChromiumPort. option_group_definitions.append(("Chromium-specific Options", [ optparse.make_option("--startup-dialog", action="store_true", default=False, help="create a dialog on DumpRenderTree startup"), optparse.make_option("--gp-fault-error-box", action="store_true", default=False, help="enable Windows GP fault error box"), optparse.make_option("--js-flags", type="string", help="JavaScript flags to pass to tests"), optparse.make_option("--stress-opt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option("--stress-deopt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option("--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option("--accelerated-video", action="store_true", help="Use hardware-accelerated compositing for video"), optparse.make_option("--no-accelerated-video", action="store_false", dest="accelerated_video", help="Don't use hardware-accelerated compositing for video"), optparse.make_option("--threaded-compositing", action="store_true", help="Use threaded compositing for rendering"), optparse.make_option("--accelerated-2d-canvas", action="store_true", help="Use hardware-accelerated 2D Canvas calls"), optparse.make_option("--no-accelerated-2d-canvas", action="store_false", dest="accelerated_2d_canvas", help="Don't use hardware-accelerated 2D Canvas calls"), optparse.make_option("--accelerated-painting", action="store_true", default=False, help="Use hardware accelerated painting of composited pages"), optparse.make_option("--enable-hardware-gpu", action="store_true", default=False, help="Run graphics tests on real GPU hardware vs software"), optparse.make_option("--per-tile-painting", action="store_true", help="Use per-tile painting of composited pages"), optparse.make_option("--adb-args", type="string", help="Arguments parsed to Android adb, to select device, etc."), ])) option_group_definitions.append(("WebKit Options", [ optparse.make_option("--gc-between-tests", action="store_true", default=False, help="Force garbage collection between each test"), optparse.make_option("--complex-text", action="store_true", default=False, help="Use the complex text code path for all text (Mac OS X and Windows only)"), optparse.make_option("-l", "--leaks", action="store_true", default=False, help="Enable leaks checking (Mac OS X only)"), optparse.make_option("-g", "--guard-malloc", action="store_true", default=False, help="Enable Guard Malloc (Mac OS X only)"), optparse.make_option("--threaded", action="store_true", default=False, help="Run a concurrent JavaScript thread with each test"), optparse.make_option("--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree."), optparse.make_option("--root", action="store", help="Path to a pre-built root of WebKit (for running tests using a nightly build of WebKit)"), ])) option_group_definitions.append(("ORWT Compatibility Options", [ # FIXME: Remove this option once the bots don't refer to it. # results.html is smart enough to figure this out itself. _compat_shim_option("--use-remote-links-to-tests"), ])) option_group_definitions.append(("Results Options", [ optparse.make_option("-p", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-sample-on-timeout", action="store_false", dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"), optparse.make_option("--no-ref-tests", action="store_true", dest="no_ref_tests", help="Skip all ref tests"), optparse.make_option("--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--new-baseline", action="store_true", default=False, help="Save generated results as new baselines " "into the *platform* directory, overwriting whatever's " "already there."), optparse.make_option("--reset-results", action="store_true", default=False, help="Reset expectations to the " "generated results in their existing location."), optparse.make_option("--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist"), optparse.make_option("--skip-pixel-test-if-no-baseline", action="store_true", dest="skip_pixel_test_if_no_baseline", help="Do not generate and check pixel result in the case when " "no image baseline is available for the test."), optparse.make_option("--skip-failing-tests", action="store_true", default=False, help="Skip tests that are expected to fail. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option("--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option("--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option("--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option("--additional-expectations", action="append", default=[], help="Path to a test_expectations file that will override previous expectations. " "Specify multiple times for multiple sets of overrides."), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), # FIXME: We should have a helper function to do this sort of # deprectated mapping and automatically log, etc. optparse.make_option("--noshow-results", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."), optparse.make_option("--no-launch-safari", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."), optparse.make_option("--full-results-html", action="store_true", default=False, help="Show all failures in results.html, rather than only regressions"), optparse.make_option("--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option("--no-record-results", action="store_false", default=True, dest="record_results", help="Don't record the results."), optparse.make_option("--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), optparse.make_option("--ignore-metrics", action="store_true", dest="ignore_metrics", default=False, help="Ignore rendering metrics related information from test " "output, only compare the structure of the rendertree."), ])) option_group_definitions.append(("Testing Options", [ optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option("-n", "--dry-run", action="store_true", default=False, help="Do everything but actually run the tests or upload results."), # old-run-webkit-tests has --valgrind instead of wrapper. optparse.make_option("--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), # old-run-webkit-tests: # -i|--ignore-tests Comma-separated list of directories # or tests to ignore optparse.make_option("-i", "--ignore-tests", action="append", default=[], help="directories or test to ignore (may specify multiple times)"), optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), # old-run-webkit-tests uses --skipped==[default|ignore|only] # instead of --force: optparse.make_option("--force", action="store_true", default=False, help="Run all tests, even those marked SKIP in the test list"), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), # old-run-webkit-tests calls --randomize-order --random: optparse.make_option("--randomize-order", action="store_true", default=False, help=("Run tests in random order (useful " "for tracking down corruption)")), optparse.make_option("--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option("--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), # old-run-webkit-tests calls --batch-size: --nthly n # Restart DumpRenderTree every n tests (default: 1000) optparse.make_option("--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), # old-run-webkit-tests calls --run-singly: -1|--singly # Isolate each test case run (implies --nthly 1 --verbose) optparse.make_option("--run-singly", action="store_true", default=False, help="run a separate DumpRenderTree for each test"), optparse.make_option("--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option("--exit-after-n-failures", type="int", default=500, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", default=20, help="Exit after the first N crashes instead of " "running all tests"), optparse.make_option("--iterations", type="int", help="Number of times to run the set of tests (e.g. ABCABCABC)"), optparse.make_option("--repeat-each", type="int", help="Number of times to run each test (e.g. AAABBBCCC)"), optparse.make_option("--retry-failures", action="store_true", default=True, help="Re-try any tests that produce unexpected results (default)"), optparse.make_option("--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results."), optparse.make_option("--max-locked-shards", type="int", help="Set the maximum number of locked shards"), ])) option_group_definitions.append(("Miscellaneous Options", [ optparse.make_option("--lint-test-files", action="store_true", default=False, help=("Makes sure the test files parse for all " "configurations. Does not run any tests.")), ])) # FIXME: Move these into json_results_generator.py option_group_definitions.append(("Result JSON Options", [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option("--builder-name", default="", help=("The name of the builder shown on the waterfall running " "this script e.g. WebKit.")), optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", help=("The name of the builder used in its path, e.g. " "webkit-rel.")), optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running this script.")), optparse.make_option("--test-results-server", default="", help=("If specified, upload results json files to this appengine " "server.")), ])) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)
def parse_args(args): option_group_definitions = [] option_group_definitions.append(("Platform options", platform_options())) option_group_definitions.append( ("Configuration options", configuration_options())) option_group_definitions.append( ("Printing Options", printing.print_options())) option_group_definitions.append(("EFL-specific Options", [ optparse.make_option( "--webprocess-cmd-prefix", type="string", default=False, help="Prefix used when spawning the Web process (Debug mode only)" ), ])) option_group_definitions.append(("Feature Switches", [ optparse.make_option( "--complex-text", action="store_true", default=False, help= "Use the complex text code path for all text (OS X and Windows only)" ), optparse.make_option("--accelerated-drawing", action="store_true", default=False, help="Use accelerated drawing (OS X only)"), optparse.make_option( "--remote-layer-tree", action="store_true", default=False, help="Use the remote layer tree drawing model (OS X WebKit2 only)" ), ])) option_group_definitions.append(( "WebKit Options", [ optparse.make_option( "--gc-between-tests", action="store_true", default=False, help="Force garbage collection between each test"), optparse.make_option( "-l", "--leaks", action="store_true", default=False, help="Enable leaks checking (OS X and Gtk+ only)"), optparse.make_option("-g", "--guard-malloc", action="store_true", default=False, help="Enable Guard Malloc (OS X only)"), optparse.make_option( "--threaded", action="store_true", default=False, help="Run a concurrent JavaScript thread with each test"), optparse.make_option( "--dump-render-tree", "-1", action="store_false", default=True, dest="webkit_test_runner", help="Use DumpRenderTree rather than WebKitTestRunner."), # FIXME: We should merge this w/ --build-directory and only have one flag. optparse.make_option( "--root", action="store", help= "Path to a directory containing the executables needed to run tests." ), ])) option_group_definitions.append(( "Results Options", [ optparse.make_option("-p", "--pixel", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-pixel", "--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-sample-on-timeout", action="store_false", default=True, dest="sample_on_timeout", help="Don't run sample on timeout (OS X only)"), optparse.make_option("--no-ref-tests", action="store_true", dest="no_ref_tests", help="Skip all ref tests"), optparse.make_option( "--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option( "--build-directory", help= "Path to the directory under which build files are kept (should not include configuration)" ), optparse.make_option( "--add-platform-exceptions", action="store_true", default=False, help= "Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory" ), optparse.make_option( "--new-baseline", action="store_true", default=False, help="Save generated results as new baselines " "into the *most-specific-platform* directory, overwriting whatever's " "already there. Equivalent to --reset-results --add-platform-exceptions" ), optparse.make_option( "--reset-results", action="store_true", default=False, help="Reset expectations to the " "generated results in their existing location."), optparse.make_option( "--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist" ), optparse.make_option( "--treat-ref-tests-as-pixel-tests", action="store_true", default=False, help= "Run ref tests, but treat them as if they were traditional pixel tests" ), #FIXME: we should support a comma separated list with --pixel-test-directory as well. optparse.make_option( "--pixel-test-directory", action="append", default=[], dest="pixel_test_directories", help= "A directory where it is allowed to execute tests as pixel tests. " "Specify multiple times to add multiple directories. " "This option implies --pixel-tests. If specified, only those tests " "will be executed as pixel tests that are located in one of the " "directories enumerated with the option. Some ports may ignore this " "option while others can have a default value that can be overridden here." ), optparse.make_option( "--skip-failing-tests", action="store_true", default=False, help="Skip tests that are expected to fail. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option( "--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option( "--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option( "--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option( "--additional-expectations", action="append", default=[], help= "Path to a test_expectations file that will override previous expectations. " "Specify multiple times for multiple sets of overrides."), optparse.make_option( "--compare-port", action="store", default=None, help="Use the specified port's baselines first"), optparse.make_option( "--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), optparse.make_option( "--full-results-html", action="store_true", default=False, help= "Show all failures in results.html, rather than only regressions" ), optparse.make_option( "--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option( "--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), optparse.make_option( "--ignore-metrics", action="store_true", dest="ignore_metrics", default=False, help="Ignore rendering metrics related information from test " "output, only compare the structure of the rendertree."), optparse.make_option( "--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option("--java", action="store_true", default=False, help="Build java support files"), optparse.make_option( "--layout-tests-directory", action="store", default=None, help="Override the default layout test directory.", dest="layout_tests_dir") ])) option_group_definitions.append(( "Testing Options", [ optparse.make_option( "--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option( "-n", "--dry-run", action="store_true", default=False, help= "Do everything but actually run the tests or upload results."), optparse.make_option( "--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree or WebKitTestRunner; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), optparse.make_option( "-i", "--ignore-tests", action="append", default=[], help= "directories or test to ignore (may specify multiple times)"), optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), optparse.make_option( "--skipped", action="store", default="default", help= ("control how tests marked SKIP are run. " "'default' == Skip tests unless explicitly listed on the command line, " "'ignore' == Run them anyway, " "'only' == only run the SKIP tests, " "'always' == always skip, even if listed on the command line." )), optparse.make_option( "--force", action="store_true", default=False, help= "Run all tests with PASS as expected result, even those marked SKIP in the test list (implies --skipped=ignore)" ), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), optparse.make_option( "--order", action="store", default="natural", help= ("determine the order in which the test cases will be run. " "'none' == use the order in which the tests were listed either in arguments or test list, " "'natural' == use the natural order (default), " "'random' == randomize the test order.")), optparse.make_option( "--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option( "--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), optparse.make_option( "--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), optparse.make_option( "--run-singly", action="store_true", default=False, help= "run a separate DumpRenderTree for each test (implies --verbose)" ), optparse.make_option( "--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option( "--exit-after-n-failures", type="int", default=None, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option( "--exit-after-n-crashes-or-timeouts", type="int", default=None, help="Exit after the first N crashes instead of " "running all tests"), optparse.make_option( "--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)" ), optparse.make_option( "--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"), optparse.make_option( "--retry-failures", action="store_true", default=True, help= "Re-try any tests that produce unexpected results (default)"), optparse.make_option( "--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results." ), optparse.make_option( "--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards"), optparse.make_option( "--additional-env-var", type="string", action="append", default=[], help= "Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)" ), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option( "--profiler", action="store", help= "Output per-test profile information, using the specified profiler." ), optparse.make_option("--no-timeout", action="store_true", default=False, help="Disable test timeouts"), optparse.make_option( "--wayland", action="store_true", default=False, help= "Run the layout tests inside a (virtualized) weston compositor (GTK only)." ), ])) option_group_definitions.append(("iOS Simulator Options", [ optparse.make_option( '--runtime', help='iOS Simulator runtime identifier (default: latest runtime)'), optparse.make_option( '--device-type', help= 'iOS Simulator device type identifier (default: i386 -> iPhone 5, x86_64 -> iPhone 5s)' ), ])) option_group_definitions.append(("Miscellaneous Options", [ optparse.make_option("--lint-test-files", action="store_true", default=False, help=("Makes sure the test files parse for all " "configurations. Does not run any tests.")), optparse.make_option( "--print-expectations", action="store_true", default=False, help= ("Print the expected outcome for the given test, or all tests listed in TestExpectations. " "Does not run any tests.")), ])) option_group_definitions.append(("Web Platform Test Server Options", [ optparse.make_option( "--wptserver-doc-root", type="string", help= ("Set web platform server document root, relative to LayoutTests directory" )), ])) # FIXME: Move these into json_results_generator.py option_group_definitions.append(("Result JSON Options", [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option( "--builder-name", default="", help= ("The name of the builder shown on the waterfall running this script. e.g. Apple MountainLion Release WK2 (Tests)." )), optparse.make_option( "--build-name", default="DUMMY_BUILD_NAME", help=( "The name of the builder used in its path, e.g. webkit-rel.")), optparse.make_option( "--build-slave", default="DUMMY_BUILD_SLAVE", help=("The name of the buildslave used. e.g. apple-macpro-6.")), optparse.make_option( "--build-number", default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running this script.")), optparse.make_option( "--test-results-server", default="", help= ("If specified, upload results json files to this appengine server." )), optparse.make_option( "--results-server-host", default="", help=( "If specified, upload results JSON file to this results server." )), optparse.make_option( "--additional-repository-name", help=("The name of an additional subversion or git checkout")), optparse.make_option( "--additional-repository-path", help= ("The path to an additional subversion or git checkout (requires --additional-repository-name)" )), optparse.make_option( "--allowed-host", type="string", action="append", default=[], help= ("If specified, tests are allowed to make requests to the specified hostname." )) ])) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)
def parse_args(args): option_group_definitions = [] option_group_definitions.append(("Platform options", platform_options())) option_group_definitions.append(("Configuration options", configuration_options())) option_group_definitions.append(("Printing Options", printing.print_options())) option_group_definitions.append( ( "Android-specific Options", [ optparse.make_option( "--adb-device", action="append", default=[], help="Run Android layout tests on these devices." ), # FIXME: Flip this to be off by default once we can log the # device setup more cleanly. optparse.make_option( "--no-android-logging", dest="android_logging", action="store_false", default=True, help=( "Do not log android-specific debug messages (default is to log as part " "of --debug-rwt-logging" ), ), ], ) ) option_group_definitions.append( ( "Results Options", [ optparse.make_option( "--add-platform-exceptions", action="store_true", default=False, help=( "Save generated results into the *most-specific-platform* directory rather " "than the *generic-platform* directory" ), ), optparse.make_option( "--additional-driver-flag", "--additional-drt-flag", dest="additional_driver_flag", action="append", default=[], help=( "Additional command line flag to pass to the driver. Specify multiple " "times to add multiple flags." ), ), optparse.make_option( "--additional-expectations", action="append", default=[], help=( "Path to a test_expectations file that will override previous " "expectations. Specify multiple times for multiple sets of overrides." ), ), optparse.make_option( "--additional-platform-directory", action="append", default=[], help=( "Additional directory where to look for test baselines (will take " "precedence over platform baselines). Specify multiple times to add " "multiple search path entries." ), ), optparse.make_option( "--build-directory", help=( "Path to the directory under which build files are kept (should not " "include configuration)" ), ), optparse.make_option( "--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs.", ), optparse.make_option( "--compare-port", action="store", default=None, help="Use the specified port's baselines first" ), optparse.make_option("--driver-name", type="string", help="Alternative driver binary to use"), optparse.make_option( "--full-results-html", action="store_true", default=False, help="Show all failures in results.html, rather than only regressions", ), optparse.make_option( "--json-test-results", action="store", help="Path to write the JSON test results to." ), optparse.make_option( "--new-baseline", action="store_true", default=False, help=( "Save generated results as new baselines into the *most-specific-platform* " "directory, overwriting whatever's already there. Equivalent to " "--reset-results --add-platform-exceptions" ), ), # TODO(ojan): Remove once bots stop using it. optparse.make_option( "--no-new-test-results", help="This doesn't do anything. TODO(ojan): Remove once bots stop using it.", ), optparse.make_option( "--new-test-results", action="store_true", default=False, help="Create new baselines when no expected results exist", ), optparse.make_option( "--no-show-results", dest="show_results", action="store_false", default=True, help="Don't launch a browser with results after the tests are done", ), optparse.make_option( "-p", "--pixel", "--pixel-tests", dest="pixel_tests", action="store_true", help="Enable pixel-to-pixel PNG comparisons", ), optparse.make_option( "--no-pixel", "--no-pixel-tests", dest="pixel_tests", action="store_false", help="Disable pixel-to-pixel PNG comparisons", ), # FIXME: we should support a comma separated list with # --pixel-test-directory as well. optparse.make_option( "--pixel-test-directory", dest="pixel_test_directories", action="append", default=[], help=( "A directory where it is allowed to execute tests as pixel tests. Specify " "multiple times to add multiple directories. This option implies " "--pixel-tests. If specified, only those tests will be executed as pixel " "tests that are located in one of the" " directories enumerated with the " "option. Some ports may ignore this option while others can have a default " "value that can be overridden here." ), ), optparse.make_option( "--reset-results", action="store_true", default=False, help="Reset expectations to the generated results in their existing location.", ), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option( "--skip-failing-tests", action="store_true", default=False, help=( "Skip tests that are expected to fail. Note: When using this option, " "you might miss new crashes in these tests." ), ), optparse.make_option("--smoke", action="store_true", help="Run just the SmokeTests"), optparse.make_option( "--no-smoke", dest="smoke", action="store_false", help="Do not run just the SmokeTests" ), ], ) ) option_group_definitions.append( ( "Testing Options", [ optparse.make_option( "--additional-env-var", type="string", action="append", default=[], help=("Passes that environment variable to the tests " "(--additional-env-var=NAME=VALUE)"), ), optparse.make_option( "--batch-size", type="int", default=None, help=("Run a the tests in batches (n), after every n tests, the driver is " "relaunched."), ), optparse.make_option( "--build", dest="build", action="store_true", default=True, help=("Check to ensure the build is up-to-date (default)."), ), optparse.make_option( "--no-build", dest="build", action="store_false", help="Don't check to see if the build is up-to-date.", ), optparse.make_option("--child-processes", help="Number of drivers to run in parallel."), optparse.make_option( "--enable-wptserve", dest="enable_wptserve", action="store_true", default=False, help="Enable running web-platform-tests using WPTserve instead of Apache.", ), optparse.make_option( "--disable-breakpad", action="store_true", help="Don't use breakpad to symbolize unexpected crashes.", ), optparse.make_option( "--driver-logging", action="store_true", help="Print detailed logging of the driver/content_shell" ), optparse.make_option( "--enable-leak-detection", action="store_true", help="Enable the leak detection of DOM objects." ), optparse.make_option( "--enable-sanitizer", action="store_true", help="Only alert on sanitizer-related errors and crashes" ), optparse.make_option( "--exit-after-n-crashes-or-timeouts", type="int", default=None, help="Exit after the first N crashes instead of running all tests", ), optparse.make_option( "--exit-after-n-failures", type="int", default=None, help="Exit after the first N failures instead of running all tests", ), optparse.make_option( "--ignore-builder-category", action="store", help=( "The category of builders to use with the --ignore-flaky-tests option " "('layout' or 'deps')." ), ), optparse.make_option( "--ignore-flaky-tests", action="store", help=( "Control whether tests that are flaky on the bots get ignored. " "'very-flaky' == Ignore any tests that flaked more than once on the bot. " "'maybe-flaky' == Ignore any tests that flaked once on the bot. " "'unexpected' == Ignore any tests that had unexpected results on the bot." ), ), optparse.make_option( "--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)", ), optparse.make_option( "--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards" ), optparse.make_option( "--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)", ), optparse.make_option( "--order", action="store", default="natural", help=( "determine the order in which the test cases will be run. " "'none' == use the order in which the tests were listed " "either in arguments or test list, " "'natural' == use the natural order (default), " "'random-seeded' == randomize the test order using a fixed seed, " "'random' == randomize the test order." ), ), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option( "--profiler", action="store", help="Output per-test profile information, using the specified profiler.", ), optparse.make_option( "--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)" ), # TODO(joelo): Delete --retry-failures and --no-retry-failures as they # are redundant with --num-retries. optparse.make_option( "--retry-failures", action="store_true", help=( "Re-try any tests that produce unexpected results. Default is to not retry " "if an explicit list of tests is passed to run-webkit-tests." ), ), optparse.make_option( "--no-retry-failures", dest="retry_failures", action="store_false", help="Don't re-try any tests that produce unexpected results.", ), optparse.make_option( "--num-retries", type="int", default=3, help=( "Number of times to retry failures, default is 3. Only relevant when " "failure retries are enabled." ), ), optparse.make_option( "--run-chunk", help="Run a specified chunk (n:l), the nth of len l, of the layout tests" ), optparse.make_option( "--run-part", help="Run a specified part (n:m), the nth of m parts, of the layout tests" ), optparse.make_option( "--run-singly", action="store_true", default=False, help="DEPRECATED, same as --batch-size=1 --verbose", ), optparse.make_option( "--skipped", action="store", default=None, help=( "control how tests marked SKIP are run. " "'default' == Skip tests unless explicitly listed on the command line, " "'ignore' == Run them anyway, " "'only' == only run the SKIP tests, " "'always' == always skip, even if listed on the command line." ), ), optparse.make_option( "--fastest", action="store", type="float", help="Run the N% fastest tests as well as any tests listed on the command line", ), optparse.make_option( "--test-list", action="append", metavar="FILE", help="read list of tests to run from file" ), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), optparse.make_option( "--wrapper", help=( "wrapper command to insert before invocations of the driver; option " "is split on whitespace before running. (Example: --wrapper='valgrind " "--smc-check=all')" ), ), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option( "-i", "--ignore-tests", action="append", default=[], help="directories or test to ignore (may specify multiple times)", ), optparse.make_option( "-n", "--dry-run", action="store_true", default=False, help="Do everything but actually run the tests or upload results.", ), ], ) ) # FIXME: Move these into json_results_generator.py. option_group_definitions.append( ( "Result JSON Options", [ optparse.make_option( "--build-name", default="DUMMY_BUILD_NAME", help="The name of the builder used in its path, e.g. webkit-rel.", ), optparse.make_option( "--step-name", default="webkit_tests", help="The name of the step in a build running this script." ), optparse.make_option( "--build-number", default="DUMMY_BUILD_NUMBER", help="The build number of the builder running this script.", ), optparse.make_option( "--builder-name", default="", help=("The name of the builder shown on the waterfall running this script " "e.g. WebKit."), ), optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option( "--test-results-server", default="", help="If specified, upload results json files to this appengine server.", ), optparse.make_option( "--write-full-results-to", help=("If specified, copy full_results.json from the results dir to the " "specified path."), ), ], ) ) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)
def parse_args(args=None): """Provides a default set of command line args. Returns a tuple of options, args from optparse""" # FIXME: All of these options should be stored closer to the code which # FIXME: actually uses them. configuration_options should move # FIXME: to WebKitPort and be shared across all scripts. configuration_options = [ optparse.make_option("-t", "--target", dest="configuration", help="(DEPRECATED)"), # FIXME: --help should display which configuration is default. optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION. ] print_options = printing.print_options() # FIXME: These options should move onto the ChromiumPort. chromium_options = [ optparse.make_option("--chromium", action="store_true", default=False, help="use the Chromium port"), optparse.make_option("--startup-dialog", action="store_true", default=False, help="create a dialog on DumpRenderTree startup"), optparse.make_option("--gp-fault-error-box", action="store_true", default=False, help="enable Windows GP fault error box"), optparse.make_option("--js-flags", type="string", help="JavaScript flags to pass to tests"), optparse.make_option( "--stress-opt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option( "--stress-deopt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option( "--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option( "--accelerated-compositing", action="store_true", help="Use hardware-accelerated compositing for rendering"), optparse.make_option( "--no-accelerated-compositing", action="store_false", dest="accelerated_compositing", help="Don't use hardware-accelerated compositing for rendering"), optparse.make_option("--threaded-compositing", action="store_true", help="Use threaded compositing for rendering"), optparse.make_option("--accelerated-2d-canvas", action="store_true", help="Use hardware-accelerated 2D Canvas calls"), optparse.make_option( "--no-accelerated-2d-canvas", action="store_false", dest="accelerated_2d_canvas", help="Don't use hardware-accelerated 2D Canvas calls"), optparse.make_option( "--accelerated-drawing", action="store_true", default=False, help="Use hardware accelerated drawing of composited pages"), optparse.make_option( "--enable-hardware-gpu", action="store_true", default=False, help="Run graphics tests on real GPU hardware vs software"), ] webkit_options = [ optparse.make_option( "--gc-between-tests", action="store_true", default=False, help="Force garbage collection between each test"), optparse.make_option( "--complex-text", action="store_true", default=False, help= "Use the complex text code path for all text (Mac OS X and Windows only)" ), optparse.make_option("-l", "--leaks", action="store_true", default=False, help="Enable leaks checking (Mac OS X only)"), optparse.make_option("-g", "--guard-malloc", action="store_true", default=False, help="Enable malloc guard (Mac OS X only)"), optparse.make_option( "--threaded", action="store_true", default=False, help="Run a concurrent JavaScript thread with each test"), optparse.make_option( "--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree."), ] old_run_webkit_tests_compat = [ # FIXME: Remove this option once the bots don't refer to it. # results.html is smart enough to figure this out itself. _compat_shim_option("--use-remote-links-to-tests"), ] results_options = [ # NEED for bots: --use-remote-links-to-tests Link to test files # within the SVN repository in the results. optparse.make_option("-p", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-sample-on-timeout", action="store_false", dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"), optparse.make_option( "--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option( "--build-directory", help= "Path to the directory under which build files are kept (should not include configuration)" ), optparse.make_option( "--new-baseline", action="store_true", default=False, help="Save all generated results as new baselines " "into the platform directory, overwriting whatever's " "already there."), optparse.make_option("--reset-results", action="store_true", default=False, help="Reset any existing baselines to the " "generated results"), optparse.make_option( "--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist"), optparse.make_option( "--skip-failing-tests", action="store_true", default=False, help="Skip tests that are expected to fail. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option( "--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option( "--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option( "--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), # FIXME: We should have a helper function to do this sort of # deprectated mapping and automatically log, etc. optparse.make_option("--noshow-results", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."), optparse.make_option( "--no-launch-safari", action="store_false", dest="show_results", help="old-run-webkit-tests compat, same as --noshow-results."), # old-run-webkit-tests: # --[no-]launch-safari Launch (or do not launch) Safari to display # test results (default: launch) optparse.make_option( "--full-results-html", action="store_true", default=False, help="Show all failures in results.html, rather than only " "regressions"), optparse.make_option("--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option( "--platform", help="Override the platform for expected results"), optparse.make_option("--no-record-results", action="store_false", default=True, dest="record_results", help="Don't record the results."), optparse.make_option("--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), ] test_options = [ optparse.make_option( "--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option( "-n", "--dry-run", action="store_true", default=False, help="Do everything but actually run the tests or upload results." ), # old-run-webkit-tests has --valgrind instead of wrapper. optparse.make_option( "--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), # old-run-webkit-tests: # -i|--ignore-tests Comma-separated list of directories # or tests to ignore optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), # old-run-webkit-tests uses --skipped==[default|ignore|only] # instead of --force: optparse.make_option( "--force", action="store_true", default=False, help="Run all tests, even those marked SKIP in the test list"), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), # old-run-webkit-tests calls --randomize-order --random: optparse.make_option("--randomize-order", action="store_true", default=False, help=("Run tests in random order (useful " "for tracking down corruption)")), optparse.make_option( "--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option("--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), # old-run-webkit-tests calls --batch-size: --nthly n # Restart DumpRenderTree every n tests (default: 1000) optparse.make_option( "--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), # old-run-webkit-tests calls --run-singly: -1|--singly # Isolate each test case run (implies --nthly 1 --verbose) optparse.make_option( "--run-singly", action="store_true", default=False, help="run a separate DumpRenderTree for each test"), optparse.make_option( "--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("--worker-model", action="store", default=None, help=("controls worker model. Valid values are " "'inline' and 'processes'.")), optparse.make_option("-f", "--experimental-fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option("--no-experimental-fully-parallel", action="store_false", dest="experimental_fully_parallel", help="do not run all tests in parallel"), optparse.make_option( "--exit-after-n-failures", type="int", default=500, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", default=20, help="Exit after the first N crashes instead of " "running all tests"), # FIXME: consider: --iterations n # Number of times to run the set of tests (e.g. ABCABCABC) optparse.make_option( "--retry-failures", action="store_true", default=True, help="Re-try any tests that produce unexpected results (default)"), optparse.make_option( "--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results."), ] misc_options = [ optparse.make_option("--lint-test-files", action="store_true", default=False, help=("Makes sure the test files parse for all " "configurations. Does not run any tests.")), ] # FIXME: Move these into json_results_generator.py results_json_options = [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option( "--builder-name", default="DUMMY_BUILDER_NAME", help=("The name of the builder shown on the waterfall running " "this script e.g. WebKit.")), optparse.make_option( "--build-name", default="DUMMY_BUILD_NAME", help=("The name of the builder used in its path, e.g. " "webkit-rel.")), optparse.make_option( "--build-number", default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running this script.")), optparse.make_option( "--test-results-server", default="", help=("If specified, upload results json files to this appengine " "server.")), ] option_list = (configuration_options + print_options + chromium_options + webkit_options + results_options + test_options + misc_options + results_json_options + old_run_webkit_tests_compat) option_parser = optparse.OptionParser(option_list=option_list) return option_parser.parse_args(args)
def parse_args(args=None): """Provides a default set of command line args. Returns a tuple of options, args from optparse""" # FIXME: All of these options should be stored closer to the code which # FIXME: actually uses them. configuration_options should move # FIXME: to WebKitPort and be shared across all scripts. configuration_options = [ optparse.make_option("-t", "--target", dest="configuration", help="(DEPRECATED)"), # FIXME: --help should display which configuration is default. optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION. ] print_options = printing.print_options() # FIXME: These options should move onto the ChromiumPort. chromium_options = [ optparse.make_option("--chromium", action="store_true", default=False, help="use the Chromium port"), optparse.make_option("--startup-dialog", action="store_true", default=False, help="create a dialog on DumpRenderTree startup"), optparse.make_option("--gp-fault-error-box", action="store_true", default=False, help="enable Windows GP fault error box"), optparse.make_option("--js-flags", type="string", help="JavaScript flags to pass to tests"), optparse.make_option("--stress-opt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option("--stress-deopt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option("--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option("--accelerated-compositing", action="store_true", help="Use hardware-accelerated compositing for rendering"), optparse.make_option("--no-accelerated-compositing", action="store_false", dest="accelerated_compositing", help="Don't use hardware-accelerated compositing for rendering"), optparse.make_option("--threaded-compositing", action="store_true", help="Use threaded compositing for rendering"), optparse.make_option("--accelerated-2d-canvas", action="store_true", help="Use hardware-accelerated 2D Canvas calls"), optparse.make_option("--no-accelerated-2d-canvas", action="store_false", dest="accelerated_2d_canvas", help="Don't use hardware-accelerated 2D Canvas calls"), optparse.make_option("--accelerated-drawing", action="store_true", default=False, help="Use hardware accelerated drawing of composited pages"), optparse.make_option("--enable-hardware-gpu", action="store_true", default=False, help="Run graphics tests on real GPU hardware vs software"), ] webkit_options = [ optparse.make_option("--gc-between-tests", action="store_true", default=False, help="Force garbage collection between each test"), optparse.make_option("--complex-text", action="store_true", default=False, help="Use the complex text code path for all text (Mac OS X and Windows only)"), optparse.make_option("-l", "--leaks", action="store_true", default=False, help="Enable leaks checking (Mac OS X only)"), optparse.make_option("-g", "--guard-malloc", action="store_true", default=False, help="Enable malloc guard (Mac OS X only)"), optparse.make_option("--threaded", action="store_true", default=False, help="Run a concurrent JavaScript thread with each test"), optparse.make_option("--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree."), ] old_run_webkit_tests_compat = [ # FIXME: Remove this option once the bots don't refer to it. # results.html is smart enough to figure this out itself. _compat_shim_option("--use-remote-links-to-tests"), ] results_options = [ # NEED for bots: --use-remote-links-to-tests Link to test files # within the SVN repository in the results. optparse.make_option("-p", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-sample-on-timeout", action="store_false", dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"), optparse.make_option("--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option("--build-directory", help="Path to the directory under which build files are kept (should not include configuration)"), optparse.make_option("--new-baseline", action="store_true", default=False, help="Save all generated results as new baselines " "into the platform directory, overwriting whatever's " "already there."), optparse.make_option("--reset-results", action="store_true", default=False, help="Reset any existing baselines to the " "generated results"), optparse.make_option("--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist"), optparse.make_option("--skip-failing-tests", action="store_true", default=False, help="Skip tests that are expected to fail. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option("--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option("--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), # FIXME: We should have a helper function to do this sort of # deprectated mapping and automatically log, etc. optparse.make_option("--noshow-results", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."), optparse.make_option("--no-launch-safari", action="store_false", dest="show_results", help="old-run-webkit-tests compat, same as --noshow-results."), # old-run-webkit-tests: # --[no-]launch-safari Launch (or do not launch) Safari to display # test results (default: launch) optparse.make_option("--full-results-html", action="store_true", default=False, help="Show all failures in results.html, rather than only " "regressions"), optparse.make_option("--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option("--platform", help="Override the platform for expected results"), optparse.make_option("--no-record-results", action="store_false", default=True, dest="record_results", help="Don't record the results."), optparse.make_option("--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), ] test_options = [ optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option("-n", "--dry-run", action="store_true", default=False, help="Do everything but actually run the tests or upload results."), # old-run-webkit-tests has --valgrind instead of wrapper. optparse.make_option("--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), # old-run-webkit-tests: # -i|--ignore-tests Comma-separated list of directories # or tests to ignore optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), # old-run-webkit-tests uses --skipped==[default|ignore|only] # instead of --force: optparse.make_option("--force", action="store_true", default=False, help="Run all tests, even those marked SKIP in the test list"), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), # old-run-webkit-tests calls --randomize-order --random: optparse.make_option("--randomize-order", action="store_true", default=False, help=("Run tests in random order (useful " "for tracking down corruption)")), optparse.make_option("--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option("--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), # old-run-webkit-tests calls --batch-size: --nthly n # Restart DumpRenderTree every n tests (default: 1000) optparse.make_option("--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), # old-run-webkit-tests calls --run-singly: -1|--singly # Isolate each test case run (implies --nthly 1 --verbose) optparse.make_option("--run-singly", action="store_true", default=False, help="run a separate DumpRenderTree for each test"), optparse.make_option("--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("--worker-model", action="store", default=None, help=("controls worker model. Valid values are " "'inline' and 'processes'.")), optparse.make_option("-f", "--experimental-fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option("--no-experimental-fully-parallel", action="store_false", dest="experimental_fully_parallel", help="do not run all tests in parallel"), optparse.make_option("--exit-after-n-failures", type="int", default=500, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", default=20, help="Exit after the first N crashes instead of " "running all tests"), # FIXME: consider: --iterations n # Number of times to run the set of tests (e.g. ABCABCABC) optparse.make_option("--retry-failures", action="store_true", default=True, help="Re-try any tests that produce unexpected results (default)"), optparse.make_option("--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results."), ] misc_options = [ optparse.make_option("--lint-test-files", action="store_true", default=False, help=("Makes sure the test files parse for all " "configurations. Does not run any tests.")), ] # FIXME: Move these into json_results_generator.py results_json_options = [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME", help=("The name of the builder shown on the waterfall running " "this script e.g. WebKit.")), optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", help=("The name of the builder used in its path, e.g. " "webkit-rel.")), optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running this script.")), optparse.make_option("--test-results-server", default="", help=("If specified, upload results json files to this appengine " "server.")), ] option_list = (configuration_options + print_options + chromium_options + webkit_options + results_options + test_options + misc_options + results_json_options + old_run_webkit_tests_compat) option_parser = optparse.OptionParser(option_list=option_list) return option_parser.parse_args(args)
def parse_args(args): option_group_definitions = [] option_group_definitions.append(("Platform options", platform_options())) option_group_definitions.append( ("Configuration options", configuration_options())) option_group_definitions.append( ("Printing Options", printing.print_options())) # FIXME: These options should move onto the ChromiumPort. option_group_definitions.append(("Chromium-specific Options", [ optparse.make_option( "--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option( "--adb-device", action="append", default=[], help="Run Android layout tests on these devices."), ])) option_group_definitions.append(( "Results Options", [ optparse.make_option("-p", "--pixel", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-pixel", "--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option("--no-ref-tests", action="store_true", dest="no_ref_tests", help="Skip all ref tests"), optparse.make_option( "--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option( "--build-directory", help= "Path to the directory under which build files are kept (should not include configuration)" ), optparse.make_option( "--add-platform-exceptions", action="store_true", default=False, help= "Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory" ), optparse.make_option( "--new-baseline", action="store_true", default=False, help="Save generated results as new baselines " "into the *most-specific-platform* directory, overwriting whatever's " "already there. Equivalent to --reset-results --add-platform-exceptions" ), optparse.make_option( "--reset-results", action="store_true", default=False, help="Reset expectations to the " "generated results in their existing location."), optparse.make_option( "--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist" ), #FIXME: we should support a comma separated list with --pixel-test-directory as well. optparse.make_option( "--pixel-test-directory", action="append", default=[], dest="pixel_test_directories", help= "A directory where it is allowed to execute tests as pixel tests. " "Specify multiple times to add multiple directories. " "This option implies --pixel-tests. If specified, only those tests " "will be executed as pixel tests that are located in one of the " "directories enumerated with the option. Some ports may ignore this " "option while others can have a default value that can be overridden here." ), optparse.make_option( "--skip-failing-tests", action="store_true", default=False, help="Skip tests that are expected to fail. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option( "--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option( "--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option( "--content-shell", action="store_true", help="Use Content Shell instead of DumpRenderTree"), optparse.make_option( "--dump-render-tree", action="store_true", help="Use DumpRenderTree instead of Content Shell"), optparse.make_option( "--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option( "--additional-expectations", action="append", default=[], help= "Path to a test_expectations file that will override previous expectations. " "Specify multiple times for multiple sets of overrides."), optparse.make_option( "--compare-port", action="store", default=None, help="Use the specified port's baselines first"), optparse.make_option( "--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), optparse.make_option( "--full-results-html", action="store_true", default=False, help= "Show all failures in results.html, rather than only regressions" ), optparse.make_option( "--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option( "--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), optparse.make_option( "--ignore-metrics", action="store_true", dest="ignore_metrics", default=False, help="Ignore rendering metrics related information from test " "output, only compare the structure of the rendertree."), ])) option_group_definitions.append(( "Testing Options", [ optparse.make_option( "--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option( "-n", "--dry-run", action="store_true", default=False, help= "Do everything but actually run the tests or upload results."), optparse.make_option( "--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), optparse.make_option( "-i", "--ignore-tests", action="append", default=[], help= "directories or test to ignore (may specify multiple times)"), optparse.make_option( "--ignore-flaky-tests", action="store", help= ("Control whether tests that are flaky on the bots get ignored." "'very-flaky' == Ignore any tests that flaked more than once on the bot." "'maybe-flaky' == Ignore any tests that flaked once on the bot." "'unexpected' == Ignore any tests that had unexpected results on the bot." )), optparse.make_option( "--ignore-builder-category", action="store", help= ("The category of builders to use with the --ignore-flaky-tests " "option ('layout' or 'deps').")), optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), optparse.make_option( "--skipped", action="store", default="default", help= ("control how tests marked SKIP are run. " "'default' == Skip tests unless explicitly listed on the command line, " "'ignore' == Run them anyway, " "'only' == only run the SKIP tests, " "'always' == always skip, even if listed on the command line." )), optparse.make_option( "--force", dest="skipped", action="store_const", const='ignore', help= "Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)" ), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), optparse.make_option( "--order", action="store", default="natural", help= ("determine the order in which the test cases will be run. " "'none' == use the order in which the tests were listed either in arguments or test list, " "'natural' == use the natural order (default), " "'random' == randomize the test order.")), optparse.make_option( "--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option( "--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), optparse.make_option( "--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), optparse.make_option( "--run-singly", action="store_true", default=False, help= "run a separate DumpRenderTree for each test (implies --verbose)" ), optparse.make_option( "--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option( "--exit-after-n-failures", type="int", default=None, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option( "--exit-after-n-crashes-or-timeouts", type="int", default=None, help="Exit after the first N crashes instead of " "running all tests"), optparse.make_option( "--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)" ), optparse.make_option( "--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"), optparse.make_option( "--retry-failures", action="store_true", help= "Re-try any tests that produce unexpected results. Default is to not retry if an explicit list of tests is passed to run-webkit-tests." ), optparse.make_option( "--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results." ), optparse.make_option( "--retry-crashes", action="store_true", default=False, help="Do also retry crashes if retry-failures is enabled."), optparse.make_option( "--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards"), optparse.make_option( "--additional-env-var", type="string", action="append", default=[], help= "Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)" ), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option( "--profiler", action="store", help= "Output per-test profile information, using the specified profiler." ), ])) option_group_definitions.append(("Miscellaneous Options", [ optparse.make_option("--lint-test-files", action="store_true", default=False, help=("Makes sure the test files parse for all " "configurations. Does not run any tests.")), ])) # FIXME: Move these into json_results_generator.py option_group_definitions.append(("Result JSON Options", [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option( "--builder-name", default="", help=("The name of the builder shown on the waterfall running " "this script e.g. WebKit.")), optparse.make_option( "--build-name", default="DUMMY_BUILD_NAME", help=("The name of the builder used in its path, e.g. " "webkit-rel.")), optparse.make_option( "--build-number", default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running this script.")), optparse.make_option( "--test-results-server", default="", help=("If specified, upload results json files to this appengine " "server.")), ])) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)
def parse_args(args=None): """Provides a default set of command line args. Returns a tuple of options, args from optparse""" option_group_definitions = [] option_group_definitions.append(("Configuration options", port_options())) option_group_definitions.append( ("Printing Options", printing.print_options())) # FIXME: These options should move onto the ChromiumPort. option_group_definitions.append(("Chromium-specific Options", [ optparse.make_option("--startup-dialog", action="store_true", default=False, help="create a dialog on DumpRenderTree startup"), optparse.make_option("--gp-fault-error-box", action="store_true", default=False, help="enable Windows GP fault error box"), optparse.make_option("--js-flags", type="string", help="JavaScript flags to pass to tests"), optparse.make_option( "--stress-opt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option( "--stress-deopt", action="store_true", default=False, help="Enable additional stress test to JavaScript optimization"), optparse.make_option( "--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option( "--accelerated-video", action="store_true", help="Use hardware-accelerated compositing for video"), optparse.make_option( "--no-accelerated-video", action="store_false", dest="accelerated_video", help="Don't use hardware-accelerated compositing for video"), optparse.make_option("--threaded-compositing", action="store_true", help="Use threaded compositing for rendering"), optparse.make_option("--accelerated-2d-canvas", action="store_true", help="Use hardware-accelerated 2D Canvas calls"), optparse.make_option( "--no-accelerated-2d-canvas", action="store_false", dest="accelerated_2d_canvas", help="Don't use hardware-accelerated 2D Canvas calls"), optparse.make_option( "--accelerated-painting", action="store_true", default=False, help="Use hardware accelerated painting of composited pages"), optparse.make_option("--per-tile-painting", action="store_true", help="Use per-tile painting of composited pages"), optparse.make_option( "--adb-device", action="append", default=[], help="Run Android layout tests on these devices."), ])) option_group_definitions.append(("EFL-specific Options", [ optparse.make_option( "--webprocess-cmd-prefix", type="string", default=False, help="Prefix used when spawning the Web process (Debug mode only)" ), ])) option_group_definitions.append(( "WebKit Options", [ optparse.make_option( "--gc-between-tests", action="store_true", default=False, help="Force garbage collection between each test"), optparse.make_option( "--complex-text", action="store_true", default=False, help= "Use the complex text code path for all text (Mac OS X and Windows only)" ), optparse.make_option("-l", "--leaks", action="store_true", default=False, help="Enable leaks checking (Mac OS X only)"), optparse.make_option("-g", "--guard-malloc", action="store_true", default=False, help="Enable Guard Malloc (Mac OS X only)"), optparse.make_option( "--threaded", action="store_true", default=False, help="Run a concurrent JavaScript thread with each test"), optparse.make_option( "--webkit-test-runner", "-2", action="store_true", help="Use WebKitTestRunner rather than DumpRenderTree."), # FIXME: We should merge this w/ --build-directory and only have one flag. optparse.make_option( "--root", action="store", help= "Path to a directory containing the executables needed to run tests." ), ])) option_group_definitions.append(( "ORWT Compatibility Options", [ # FIXME: Remove this option once the bots don't refer to it. # results.html is smart enough to figure this out itself. _compat_shim_option("--use-remote-links-to-tests"), ])) option_group_definitions.append(( "Results Options", [ optparse.make_option("-p", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-sample-on-timeout", action="store_false", dest="sample_on_timeout", help="Don't run sample on timeout (Mac OS X only)"), optparse.make_option("--no-ref-tests", action="store_true", dest="no_ref_tests", help="Skip all ref tests"), optparse.make_option( "--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option( "--build-directory", help= "Path to the directory under which build files are kept (should not include configuration)" ), optparse.make_option( "--add-platform-exceptions", action="store_true", default=False, help= "Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory" ), optparse.make_option( "--new-baseline", action="store_true", default=False, help="Save generated results as new baselines " "into the *most-specific-platform* directory, overwriting whatever's " "already there. Equivalent to --reset-results --add-platform-exceptions" ), optparse.make_option( "--reset-results", action="store_true", default=False, help="Reset expectations to the " "generated results in their existing location."), optparse.make_option( "--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist" ), #FIXME: we should support a comma separated list with --pixel-test-directory as well. optparse.make_option( "--pixel-test-directory", action="append", default=[], dest="pixel_test_directories", help= "A directory where it is allowed to execute tests as pixel tests. " "Specify multiple times to add multiple directories. " "This option implies --pixel-tests. If specified, only those tests " "will be executed as pixel tests that are located in one of the " "directories enumerated with the option. Some ports may ignore this " "option while others can have a default value that can be overridden here." ), optparse.make_option( "--skip-failing-tests", action="store_true", default=False, help="Skip tests that are expected to fail. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option( "--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option( "--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option( "--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option( "--additional-expectations", action="append", default=[], help= "Path to a test_expectations file that will override previous expectations. " "Specify multiple times for multiple sets of overrides."), optparse.make_option( "--compare-port", action="store", default=None, help="Use the specified port's baselines first"), optparse.make_option( "--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), # FIXME: We should have a helper function to do this sort of # deprectated mapping and automatically log, etc. optparse.make_option( "--noshow-results", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."), optparse.make_option( "--no-launch-safari", action="store_false", dest="show_results", help="Deprecated, same as --no-show-results."), optparse.make_option( "--full-results-html", action="store_true", default=False, help= "Show all failures in results.html, rather than only regressions" ), optparse.make_option( "--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option("--no-record-results", action="store_false", default=True, dest="record_results", help="Don't record the results."), optparse.make_option( "--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), optparse.make_option( "--ignore-metrics", action="store_true", dest="ignore_metrics", default=False, help="Ignore rendering metrics related information from test " "output, only compare the structure of the rendertree."), ])) option_group_definitions.append(( "Testing Options", [ optparse.make_option( "--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option( "-n", "--dry-run", action="store_true", default=False, help= "Do everything but actually run the tests or upload results."), optparse.make_option( "--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), optparse.make_option( "-i", "--ignore-tests", action="append", default=[], help= "directories or test to ignore (may specify multiple times)"), optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), optparse.make_option( "--skipped", action="store", default="default", help= "control how tests marked SKIP are run. 'default' == Skip, 'ignore' == Run them anyway, 'only' == only run the SKIP tests." ), optparse.make_option( "--force", dest="skipped", action="store_const", const='ignore', help= "Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)" ), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), optparse.make_option("--randomize-order", action="store_true", default=False, help=("Run tests in random order (useful " "for tracking down corruption)")), optparse.make_option( "--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option( "--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), optparse.make_option( "--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), optparse.make_option( "--run-singly", action="store_true", default=False, help= "run a separate DumpRenderTree for each test (implies --verbose)" ), optparse.make_option( "--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option( "--exit-after-n-failures", type="int", default=None, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option( "--exit-after-n-crashes-or-timeouts", type="int", default=None, help="Exit after the first N crashes instead of " "running all tests"), optparse.make_option( "--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)" ), optparse.make_option( "--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"), optparse.make_option( "--retry-failures", action="store_true", default=True, help= "Re-try any tests that produce unexpected results (default)"), optparse.make_option( "--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results." ), optparse.make_option( "--max-locked-shards", type="int", default=1, help="Set the maximum number of locked shards"), optparse.make_option( "--additional-env-var", type="string", action="append", default=[], help= "Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)" ), ])) option_group_definitions.append(("Miscellaneous Options", [ optparse.make_option("--lint-test-files", action="store_true", default=False, help=("Makes sure the test files parse for all " "configurations. Does not run any tests.")), ])) # FIXME: Move these into json_results_generator.py option_group_definitions.append(("Result JSON Options", [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option( "--builder-name", default="", help=("The name of the builder shown on the waterfall running " "this script e.g. WebKit.")), optparse.make_option( "--build-name", default="DUMMY_BUILD_NAME", help=("The name of the builder used in its path, e.g. " "webkit-rel.")), optparse.make_option( "--build-number", default="DUMMY_BUILD_NUMBER", help=("The build number of the builder running this script.")), optparse.make_option( "--test-results-server", default="", help=("If specified, upload results json files to this appengine " "server.")), ])) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)
def parse_args(args): option_group_definitions = [] option_group_definitions.append(("Platform options", platform_options())) option_group_definitions.append( ("Configuration options", configuration_options())) option_group_definitions.append( ("Printing Options", printing.print_options())) option_group_definitions.append(("Feature Switches", [ optparse.make_option( "--complex-text", action="store_true", default=False, help= "Use the complex text code path for all text (OS X and Windows only)" ), optparse.make_option("--accelerated-drawing", action="store_true", default=False, help="Use accelerated drawing (OS X only)"), optparse.make_option( "--remote-layer-tree", action="store_true", default=False, help="Use the remote layer tree drawing model (OS X WebKit2 only)" ), optparse.make_option( "--internal-feature", type="string", action="append", default=[], help= "Enable (disable) an internal feature (--internal-feature FeatureName[=true|false])" ), optparse.make_option( "--experimental-feature", type="string", action="append", default=[], help= "Enable (disable) an experimental feature (--experimental-feature FeatureName[=true|false])" ), ])) option_group_definitions.append(( "WebKit Options", [ optparse.make_option( "--gc-between-tests", action="store_true", default=False, help="Force garbage collection between each test"), optparse.make_option( "-l", "--leaks", action="store_true", default=False, help="Enable leaks checking (OS X and Gtk+ only)"), optparse.make_option("-g", "--guard-malloc", action="store_true", default=False, help="Enable Guard Malloc (OS X only)"), optparse.make_option( "--threaded", action="store_true", default=False, help="Run a concurrent JavaScript thread with each test"), optparse.make_option( "--dump-render-tree", "-1", action="store_false", default=True, dest="webkit_test_runner", help="Use DumpRenderTree rather than WebKitTestRunner."), # FIXME: We should merge this w/ --build-directory and only have one flag. optparse.make_option( "--root", action="store", help= "Path to a directory containing the executables needed to run tests." ), ])) option_group_definitions.append(( "Results Options", [ optparse.make_option("-p", "--pixel", "--pixel-tests", action="store_true", dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-pixel", "--no-pixel-tests", action="store_false", dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-sample-on-timeout", action="store_false", default=True, dest="sample_on_timeout", help="Don't run sample on timeout (OS X only)"), optparse.make_option("--no-ref-tests", action="store_true", dest="no_ref_tests", help="Skip all ref tests"), optparse.make_option( "--ignore-render-tree-dump-results", action="store_true", dest="ignore_render_tree_dump_results", help= "Don't compare or save results for render tree dump tests (they still run and crashes are reported)" ), optparse.make_option( "--tolerance", help="Ignore image differences less than this percentage (some " "ports may ignore this option)", type="float"), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option( "--build-directory", help= "Path to the directory under which build files are kept (should not include configuration)" ), optparse.make_option( "--add-platform-exceptions", action="store_true", default=False, help= "Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory" ), optparse.make_option( "--new-baseline", action="store_true", default=False, help="Save generated results as new baselines " "into the *most-specific-platform* directory, overwriting whatever's " "already there. Equivalent to --reset-results --add-platform-exceptions" ), optparse.make_option( "--reset-results", action="store_true", default=False, help="Reset expectations to the " "generated results in their existing location."), optparse.make_option( "--no-new-test-results", action="store_false", dest="new_test_results", default=True, help="Don't create new baselines when no expected results exist" ), optparse.make_option( "--treat-ref-tests-as-pixel-tests", action="store_true", default=False, help= "Run ref tests, but treat them as if they were traditional pixel tests" ), #FIXME: we should support a comma separated list with --pixel-test-directory as well. optparse.make_option( "--pixel-test-directory", action="append", default=[], dest="pixel_test_directories", help= "A directory where it is allowed to execute tests as pixel tests. " "Specify multiple times to add multiple directories. " "This option implies --pixel-tests. If specified, only those tests " "will be executed as pixel tests that are located in one of the " "directories enumerated with the option. Some ports may ignore this " "option while others can have a default value that can be overridden here." ), optparse.make_option( "--skip-failing-tests", action="store_true", default=False, help="Skip tests that are marked as failing or flaky. " "Note: When using this option, you might miss new crashes " "in these tests."), optparse.make_option( "--additional-drt-flag", action="append", default=[], help="Additional command line flag to pass to DumpRenderTree " "Specify multiple times to add multiple flags."), optparse.make_option( "--driver-name", type="string", help="Alternative DumpRenderTree binary to use"), optparse.make_option( "--additional-platform-directory", action="append", default=[], help="Additional directory where to look for test " "baselines (will take precendence over platform baselines). " "Specify multiple times to add multiple search path entries."), optparse.make_option( "--additional-expectations", action="append", default=[], help= "Path to a test_expectations file that will override previous expectations. " "Specify multiple times for multiple sets of overrides."), optparse.make_option( "--compare-port", action="store", default=None, help="Use the specified port's baselines first"), optparse.make_option( "--no-show-results", action="store_false", default=True, dest="show_results", help="Don't launch a browser with results after the tests " "are done"), optparse.make_option( "--full-results-html", action="store_true", default=False, help= "Show all failures in results.html, rather than only regressions" ), optparse.make_option( "--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option( "--http", action="store_true", dest="http", default=True, help="Run HTTP and WebSocket tests (default)"), optparse.make_option("--no-http", action="store_false", dest="http", help="Don't run HTTP and WebSocket tests"), optparse.make_option("--no-http-servers", action="store_false", dest="start_http_servers_if_needed", default=True, help="Don't start HTTP servers"), optparse.make_option( "--ignore-metrics", action="store_true", dest="ignore_metrics", default=False, help="Ignore rendering metrics related information from test " "output, only compare the structure of the rendertree."), optparse.make_option( "--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option("--java", action="store_true", default=False, help="Build java support files"), optparse.make_option( "--layout-tests-directory", action="store", default=None, help="Override the default layout test directory.", dest="layout_tests_dir") ])) option_group_definitions.append(("Testing Options", [ optparse.make_option("--build", dest="build", action="store_true", default=True, help="Check to ensure the DumpRenderTree build is up-to-date " "(default)."), optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), optparse.make_option("-n", "--dry-run", action="store_true", default=False, help="Do everything but actually run the tests or upload results."), optparse.make_option("--wrapper", help="wrapper command to insert before invocations of " "DumpRenderTree or WebKitTestRunner; option is split on whitespace before " "running. (Example: --wrapper='valgrind --smc-check=all')"), optparse.make_option("-i", "--ignore-tests", action="append", default=[], help="directories or test to ignore (may specify multiple times)"), optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), optparse.make_option("--skipped", action="store", default="default", help=("control how tests marked SKIP are run. " "'default' == Skip tests unless explicitly listed on the command line, " "'ignore' == Run them anyway, " "'only' == only run the SKIP tests, " "'always' == always skip, even if listed on the command line.")), optparse.make_option("--force", action="store_true", default=False, help="Run all tests with PASS as expected result, even those marked SKIP in the test list or " + \ "those which are device-specific (implies --skipped=ignore)"), optparse.make_option("--time-out-ms", "--timeout", help="Set the timeout for each test in milliseconds"), optparse.make_option("--order", action="store", default="natural", help=("determine the order in which the test cases will be run. " "'none' == use the order in which the tests were listed either in arguments or test list, " "'natural' == use the natural order (default), " "'random' == randomize the test order.")), optparse.make_option("--run-chunk", help=("Run a specified chunk (n:l), the nth of len l, " "of the layout tests")), optparse.make_option("--run-part", help=("Run a specified part (n:m), " "the nth of m parts, of the layout tests")), optparse.make_option("--batch-size", help=("Run a the tests in batches (n), after every n tests, " "DumpRenderTree is relaunched."), type="int", default=None), optparse.make_option("--run-singly", action="store_true", default=False, help="run a separate DumpRenderTree for each test (implies --verbose)"), optparse.make_option("--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option("--exit-after-n-failures", type="int", default=None, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", default=None, help="Exit after the first N crashes instead of " "running all tests"), optparse.make_option("--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)"), optparse.make_option("--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"), optparse.make_option("--retry-failures", action="store_true", default=True, help="Re-try any tests that produce unexpected results (default)"), optparse.make_option("--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results."), optparse.make_option("--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards"), optparse.make_option("--additional-env-var", type="string", action="append", default=[], help="Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)"), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option("--profiler", action="store", help="Output per-test profile information, using the specified profiler."), optparse.make_option("--no-timeout", action="store_true", default=False, help="Disable test timeouts"), optparse.make_option('--display-server', choices=['xvfb', 'xorg', 'weston', 'wayland'], default='xvfb', help='"xvfb": Use a virtualized X11 server. "xorg": Use the current X11 session. ' '"weston": Use a virtualized Weston server. "wayland": Use the current wayland session.'), optparse.make_option("--world-leaks", action="store_true", default=False, help="Check for world leaks (currently, only documents). Differs from --leaks in that this uses internal instrumentation, rather than external tools."), optparse.make_option("--accessibility-isolated-tree", action="store_true", default=False, help="Runs tests in accessibility isolated tree mode."), ])) option_group_definitions.append(("iOS Options", [ optparse.make_option( '--no-install', action='store_const', const=False, default=True, dest='install', help='Skip install step for device and simulator testing'), optparse.make_option( '--version', help= 'Specify the version of iOS to be used. By default, this will adopt the runtime for iOS Simulator.' ), optparse.make_option( '--device-type', help= 'iOS Simulator device type identifier (default: i386 -> iPhone 5, x86_64 -> iPhone SE)' ), optparse.make_option( '--dedicated-simulators', action="store_true", default=False, help= "If set, dedicated iOS simulators will always be created. If not set, the script will attempt to use any currently running simulator." ), optparse.make_option( '--show-touches', action="store_true", default=False, help= "If set, a small dot will be shown where the generated touches are. Helpful for debugging touch tests." ), ])) option_group_definitions.append(("Miscellaneous Options", [ optparse.make_option( "--lint-test-files", action="store_true", default=False, help= ("Makes sure the test files parse for all configurations. Does not run any tests." )), optparse.make_option( "--print-expectations", action="store_true", default=False, help= ("Print the expected outcome for the given test, or all tests listed in TestExpectations. Does not run any tests." )), optparse.make_option( "--webgl-test-suite", action="store_true", default=False, help= ("Run exhaustive webgl list, including test ordinarily skipped for performance reasons. Equivalent to '--additional-expectations=LayoutTests/webgl/TestExpectations webgl'" )), optparse.make_option( "--use-gpu-process", action="store_true", default=False, help= ("Enable all GPU process related features, also set additional expectations and the result report flavor." )), optparse.make_option( "--prefer-integrated-gpu", action="store_true", default=False, help= ("Prefer using the lower-power integrated GPU on a dual-GPU system. Note that other running applications and the tests themselves can override this request." )), ])) option_group_definitions.append(("Web Platform Test Server Options", [ optparse.make_option( "--wptserver-doc-root", type="string", help= ("Set web platform server document root, relative to LayoutTests directory" )), ])) # FIXME: Remove this group once the old results dashboards are deprecated. option_group_definitions.append(("Legacy Result Options", [ optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option( "--build-name", default="DUMMY_BUILD_NAME", help=( "The name of the builder used in its path, e.g. webkit-rel.")), optparse.make_option( "--build-slave", default="DUMMY_BUILD_SLAVE", help=("The name of the worker used. e.g. apple-macpro-6.")), optparse.make_option( "--test-results-server", action="append", default=[], help= ("If specified, upload results json files to this appengine server." )), optparse.make_option( "--results-server-host", action="append", default=[], help=( "If specified, upload results JSON file to this results server." )), optparse.make_option( "--additional-repository-name", help=("The name of an additional subversion or git checkout")), optparse.make_option( "--additional-repository-path", help= ("The path to an additional subversion or git checkout (requires --additional-repository-name)" )), optparse.make_option( "--allowed-host", type="string", action="append", default=[], help= ("If specified, tests are allowed to make requests to the specified hostname." )) ])) option_group_definitions.append(('Upload Options', upload_options())) option_parser = optparse.OptionParser(usage="%prog [options] [<path>...]") for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) options, args = option_parser.parse_args(args) if options.webgl_test_suite: if not args: args.append('webgl') host = Host() host.initialize_scm() options.additional_expectations.insert( 0, host.filesystem.join(host.scm().checkout_root, 'LayoutTests/webgl/TestExpectations')) if options.use_gpu_process: host = Host() host.initialize_scm() options.additional_expectations.insert( 0, host.filesystem.join(host.scm().checkout_root, 'LayoutTests/gpu-process/TestExpectations')) if not options.internal_feature: options.internal_feature = [] options.internal_feature.append('UseGPUProcessForMediaEnabled') options.internal_feature.append('CaptureAudioInGPUProcessEnabled') options.internal_feature.append('CaptureVideoInGPUProcessEnabled') options.internal_feature.append( 'UseGPUProcessForCanvasRenderingEnabled') options.internal_feature.append('UseGPUProcessForDOMRenderingEnabled') options.internal_feature.append('UseGPUProcessForWebGLEnabled') if not options.experimental_feature: options.experimental_feature = [] options.experimental_feature.append( 'WebRTCPlatformCodecsInGPUProcessEnabled') if options.result_report_flavor: raise RuntimeError( '--use-gpu-process implicitly sets the result flavor, this should not be overridden' ) options.result_report_flavor = 'gpuprocess' return options, args
def parse_args(args): option_group_definitions = [] option_group_definitions.append( ('Platform options', platform_options())) option_group_definitions.append( ('Configuration options', configuration_options())) option_group_definitions.append( ('Printing Options', printing.print_options())) option_group_definitions.append( ('Android-specific Options', [ optparse.make_option( '--adb-device', action='append', default=[], dest='adb_devices', help='Run Android layout tests on these devices'), # FIXME: Flip this to be off by default once we can log the # device setup more cleanly. optparse.make_option( '--no-android-logging', dest='android_logging', action='store_false', default=True, help=('Do not log android-specific debug messages (default ' 'is to log as part of --debug-rwt-logging)')), ])) option_group_definitions.append( ('Fuchsia-specific Options', [ optparse.make_option( '--zircon-logging', dest='zircon_logging', action='store_true', default=True, help=('Log Zircon debug messages (enabled by default).')), optparse.make_option( '--no-zircon-logging', dest='zircon_logging', action='store_false', default=True, help=('Do not log Zircon debug messages.')), ])) option_group_definitions.append( ('Results Options', [ optparse.make_option( '--add-platform-exceptions', action='callback', callback=deprecate, help=('Deprecated. Use "blink_tool.py rebaseline*" instead.')), optparse.make_option( '--additional-driver-flag', '--additional-drt-flag', dest='additional_driver_flag', action='append', default=[], help=('Additional command line flag to pass to the driver. Specify multiple ' 'times to add multiple flags.')), optparse.make_option( '--additional-expectations', action='append', default=[], help=('Path to a test_expectations file that will override previous ' 'expectations. Specify multiple times for multiple sets of overrides.')), optparse.make_option( '--additional-platform-directory', action='append', default=[], help=('Additional directory where to look for test baselines (will take ' 'precedence over platform baselines). Specify multiple times to add ' 'multiple search path entries.')), optparse.make_option( '--build-directory', default='out', help=('Path to the directory where build files are kept, not including ' 'configuration. In general this will be "out".')), optparse.make_option( '--clobber-old-results', action='store_true', default=False, help='Clobbers test results from previous runs.'), optparse.make_option( '--compare-port', action='store', default=None, help="Use the specified port's baselines first"), optparse.make_option( '--copy-baselines', action='store_true', default=False, help=('If the actual result is different from the current baseline, ' 'copy the current baseline into the *most-specific-platform* ' 'directory, or the flag-specific generic-platform directory if ' '--additional-driver-flag is specified. See --reset-results.')), optparse.make_option( '--driver-name', type='string', help='Alternative driver binary to use'), optparse.make_option( '--json-test-results', # New name from json_results_generator '--write-full-results-to', # Old argument name '--isolated-script-test-output', # Isolated API help='Path to write the JSON test results for *all* tests.'), # FIXME(tansell): Remove this option if nobody is found who needs it. optparse.make_option( '--json-failing-test-results', help='Path to write the JSON test results for only *failing* tests.'), optparse.make_option( '--new-baseline', action='callback', callback=deprecate, help=('Deprecated. Use "blink_tool.py rebaseline*" instead.')), optparse.make_option( '--new-flag-specific-baseline', action='callback', callback=deprecate, help='Deprecated. Use --copy-baselines --reset-results instead.'), optparse.make_option( '--new-test-results', action='callback', callback=deprecate, help='Deprecated. Use --reset-results instead.'), optparse.make_option( '--no-show-results', dest='show_results', action='store_false', default=True, help="Don't launch a browser with results after the tests are done"), optparse.make_option( '-p', '--pixel', '--pixel-tests', dest='pixel_tests', action='store_true', default=True, help='Enable pixel-to-pixel PNG comparisons (enabled by default)'), optparse.make_option( '--no-pixel', '--no-pixel-tests', dest='pixel_tests', action='store_false', default=True, help='Disable pixel-to-pixel PNG comparisons'), # FIXME: we should support a comma separated list with # --pixel-test-directory as well. optparse.make_option( '--pixel-test-directory', dest='pixel_test_directories', action='append', default=[], help=('A directory where it is allowed to execute tests as pixel tests. Specify ' 'multiple times to add multiple directories. This option implies ' '--pixel-tests. If specified, only those tests will be executed as pixel ' 'tests that are located in one of the' ' directories enumerated with the ' 'option. Some ports may ignore this option while others can have a default ' 'value that can be overridden here.')), optparse.make_option( '--reset-results', action='store_true', default=False, help=('Reset baselines to the generated results in their existing location or the default ' 'location if no baseline exists. For virtual tests, reset the virtual baselines. ' 'If --additional-driver-flag is specified, reset the flag-specific baselines. ' 'If --copy-baselines is specified, the copied baselines will be reset.')), optparse.make_option( '--results-directory', help='Location of test results'), optparse.make_option( '--smoke', action='store_true', help='Run just the SmokeTests'), optparse.make_option( '--no-smoke', dest='smoke', action='store_false', help='Do not run just the SmokeTests'), optparse.make_option( '--image-first-tests', action='append', default=[], dest='image_first_tests', help=('A directory (or test) where the test result will only be compared with the ' 'image baseline if an image baseline is available, and fall back to comparison ' 'with the text baseline when image baselines are missing. Specify multiple times ' 'to add multiple directories/tests.')), ])) option_group_definitions.append( ('Testing Options', [ optparse.make_option( '--additional-env-var', type='string', action='append', default=[], help=('Passes that environment variable to the tests ' '(--additional-env-var=NAME=VALUE)')), optparse.make_option( '--batch-size', type='int', default=None, help=('Run a the tests in batches (n), after every n tests, the driver is ' 'relaunched.')), optparse.make_option( '--build', dest='build', action='store_true', default=True, help=('Check to ensure the build is up to date (default).')), optparse.make_option( '--no-build', dest='build', action='store_false', help="Don't check to see if the build is up to date."), optparse.make_option( '--child-processes', help='Number of drivers to run in parallel.'), optparse.make_option( '--disable-breakpad', action='store_true', help="Don't use breakpad to symbolize unexpected crashes."), optparse.make_option( '--driver-logging', action='store_true', help='Print detailed logging of the driver/content_shell'), optparse.make_option( '--enable-leak-detection', action='store_true', help='Enable the leak detection of DOM objects.'), optparse.make_option( '--enable-sanitizer', action='store_true', help='Only alert on sanitizer-related errors and crashes'), optparse.make_option( '--exit-after-n-crashes-or-timeouts', type='int', default=None, help='Exit after the first N crashes instead of running all tests'), optparse.make_option( '--exit-after-n-failures', type='int', default=None, help='Exit after the first N failures instead of running all tests'), optparse.make_option( '--ignore-builder-category', action='store', help=('The category of builders to use with the --ignore-flaky-tests option ' "('layout' or 'deps').")), optparse.make_option( '--ignore-flaky-tests', action='store', help=('Control whether tests that are flaky on the bots get ignored. ' "'very-flaky' == Ignore any tests that flaked more than once on the bot. " "'maybe-flaky' == Ignore any tests that flaked once on the bot. " "'unexpected' == Ignore any tests that had unexpected results on the bot.")), optparse.make_option( '--iterations', type='int', default=1, help='Number of times to run the set of tests (e.g. ABCABCABC)'), optparse.make_option( '--layout-tests-directory', help=('Path to a custom layout tests directory')), optparse.make_option( '--max-locked-shards', type='int', default=0, help='Set the maximum number of locked shards'), optparse.make_option( '--nocheck-sys-deps', action='store_true', default=False, help="Don't check the system dependencies (themes)"), optparse.make_option( '--order', action='store', default='random', help=('Determine the order in which the test cases will be run. ' "'none' == use the order in which the tests were listed " 'either in arguments or test list, ' "'random' == pseudo-random order (default). Seed can be specified " 'via --seed, otherwise it will default to the current unix timestamp. ' "'natural' == use the natural order")), optparse.make_option( '--profile', action='store_true', help='Output per-test profile information.'), optparse.make_option( '--profiler', action='store', help='Output per-test profile information, using the specified profiler.'), optparse.make_option( '--repeat-each', '--gtest_repeat', type='int', default=1, help='Number of times to run each test (e.g. AAABBBCCC)'), optparse.make_option( '--num-retries', '--test-launcher-retry-limit', type='int', default=None, help=('Number of times to retry failures. Default (when this ' 'flag is not specified) is to retry 3 times, unless an ' 'explicit list of tests is passed to run-webkit-tests. ' 'If a non-zero value is given explicitly, failures are ' 'retried regardless.')), optparse.make_option( '--no-retry-failures', dest='num_retries', action='store_const', const=0, help="Don't retry any failures (equivalent to --num-retries=0)."), optparse.make_option( '--total-shards', type=int, help=('Total number of shards being used for this test run. ' 'Must be used with --shard-index. ' '(The user of this script is responsible for spawning ' 'all of the shards.)')), optparse.make_option( '--shard-index', type=int, help=('Shard index [0..total_shards) of this test run. ' 'Must be used with --total-shards.')), optparse.make_option( '--run-singly', action='store_true', default=False, help='DEPRECATED, same as --batch-size=1 --verbose'), optparse.make_option( '--seed', type='int', help=('Seed to use for random test order (default: %default). ' 'Only applicable in combination with --order=random.')), optparse.make_option( '--skipped', action='store', default=None, help=('Control how tests marked SKIP are run. ' '"default" == Skip tests unless explicitly listed on the command line, ' '"ignore" == Run them anyway, ' '"only" == only run the SKIP tests, ' '"always" == always skip, even if listed on the command line.')), optparse.make_option( '--gtest_also_run_disabled_tests', action='store_true', default=False, # Consistent with the default value of --skipped help=('Equivalent to --skipped=ignore. This option overrides ' '--skipped if both are given.')), optparse.make_option( '--skip-failing-tests', action='store_true', default=False, help=('Skip tests that are expected to fail. Note: When using this option, ' 'you might miss new crashes in these tests.')), optparse.make_option( '--skip-timeouts', action='store_true', default=False, help=('Skip tests marked TIMEOUT. Use it to speed up running the entire ' 'test suite.')), optparse.make_option( '--fastest', action='store', type='float', help='Run the N% fastest tests as well as any tests listed on the command line'), optparse.make_option( '--test-list', action='append', metavar='FILE', help='read list of tests to run from file'), optparse.make_option( '--gtest_filter', type='string', help='A colon-separated list of tests to run. Wildcards are ' 'NOT supported. It is the same as listing the tests as ' 'positional arguments.'), optparse.make_option( '--time-out-ms', help='Set the timeout for each test'), optparse.make_option( '--wrapper', help=('wrapper command to insert before invocations of the driver; option ' 'is split on whitespace before running. (Example: --wrapper="valgrind ' '--smc-check=all")')), # FIXME: Display the default number of child processes that will run. optparse.make_option( '-f', '--fully-parallel', action='store_true', help='run all tests in parallel'), optparse.make_option( '-i', '--ignore-tests', action='append', default=[], help='directories or test to ignore (may specify multiple times)'), optparse.make_option( '-n', '--dry-run', action='store_true', default=False, help='Do everything but actually run the tests or upload results.'), optparse.make_option( '-w', '--watch', action='store_true', help='Re-run tests quickly (e.g. avoid restarting the server)'), optparse.make_option( '--zero-tests-executed-ok', action='store_true', help='If set, exit with a success code when no tests are run.' ' Used on trybots when layout tests are retried without patch.') ])) # FIXME: Move these into json_results_generator.py. option_group_definitions.append( ('Result JSON Options', [ optparse.make_option( '--build-name', default='DUMMY_BUILD_NAME', help='The name of the builder used in its path, e.g. webkit-rel.'), optparse.make_option( '--step-name', default='webkit_tests', help='The name of the step in a build running this script.'), optparse.make_option( '--build-number', default='DUMMY_BUILD_NUMBER', help='The build number of the builder running this script.'), optparse.make_option( '--builder-name', default='', help=('The name of the builder shown on the waterfall running this script ' 'e.g. WebKit.')), optparse.make_option( '--master-name', help='The name of the buildbot master.'), optparse.make_option( '--test-results-server', default='', help='If specified, upload results json files to this appengine server.'), ])) option_parser = optparse.OptionParser( prog='run-webkit-tests', usage='%prog [options] [tests]', description='Runs Blink layout tests as described in docs/testing/layout_tests.md') for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) (options, args) = option_parser.parse_args(args) return (options, args)
def parse_args(args): option_group_definitions = [] option_group_definitions.append(("Platform options", platform_options())) option_group_definitions.append( ("Configuration options", configuration_options())) option_group_definitions.append( ("Printing Options", printing.print_options())) option_group_definitions.append(( "Android-specific Options", [ optparse.make_option( "--adb-device", action="append", default=[], help="Run Android layout tests on these devices."), # FIXME: Flip this to be off by default once we can log the # device setup more cleanly. optparse.make_option( "--no-android-logging", dest="android_logging", action="store_false", default=True, help= ("Do not log android-specific debug messages (default is to log as part " "of --debug-rwt-logging")), ])) option_group_definitions.append(( "Results Options", [ optparse.make_option( "--add-platform-exceptions", action="store_true", default=False, help= ("Save generated results into the *most-specific-platform* directory rather " "than the *generic-platform* directory")), optparse.make_option( "--additional-driver-flag", "--additional-drt-flag", dest="additional_driver_flag", action="append", default=[], help= ("Additional command line flag to pass to the driver. Specify multiple " "times to add multiple flags.")), optparse.make_option( "--additional-expectations", action="append", default=[], help= ("Path to a test_expectations file that will override previous " "expectations. Specify multiple times for multiple sets of overrides." )), optparse.make_option( "--additional-platform-directory", action="append", default=[], help= ("Additional directory where to look for test baselines (will take " "precedence over platform baselines). Specify multiple times to add " "multiple search path entries.")), optparse.make_option( "--build-directory", help= ("Path to the directory under which build files are kept (should not " "include configuration)")), optparse.make_option( "--clobber-old-results", action="store_true", default=False, help="Clobbers test results from previous runs."), optparse.make_option( "--compare-port", action="store", default=None, help="Use the specified port's baselines first"), optparse.make_option("--driver-name", type="string", help="Alternative driver binary to use"), optparse.make_option( "--full-results-html", action="store_true", default=False, help= "Show all failures in results.html, rather than only regressions" ), optparse.make_option( "--json-test-results", action="store", help="Path to write the JSON test results to."), optparse.make_option( "--new-baseline", action="store_true", default=False, help= ("Save generated results as new baselines into the *most-specific-platform* " "directory, overwriting whatever's already there. Equivalent to " "--reset-results --add-platform-exceptions")), # TODO(ojan): Remove once bots stop using it. optparse.make_option( "--no-new-test-results", help= "This doesn't do anything. TODO(ojan): Remove once bots stop using it." ), optparse.make_option( "--new-test-results", action="store_true", default=False, help="Create new baselines when no expected results exist"), optparse.make_option( "--no-show-results", dest="show_results", action="store_false", default=True, help= "Don't launch a browser with results after the tests are done" ), optparse.make_option("-p", "--pixel", "--pixel-tests", dest="pixel_tests", action="store_true", help="Enable pixel-to-pixel PNG comparisons"), optparse.make_option( "--no-pixel", "--no-pixel-tests", dest="pixel_tests", action="store_false", help="Disable pixel-to-pixel PNG comparisons"), # FIXME: we should support a comma separated list with # --pixel-test-directory as well. optparse.make_option( "--pixel-test-directory", dest="pixel_test_directories", action="append", default=[], help= ("A directory where it is allowed to execute tests as pixel tests. Specify " "multiple times to add multiple directories. This option implies " "--pixel-tests. If specified, only those tests will be executed as pixel " "tests that are located in one of the" " directories enumerated with the " "option. Some ports may ignore this option while others can have a default " "value that can be overridden here.")), optparse.make_option( "--reset-results", action="store_true", default=False, help= "Reset expectations to the generated results in their existing location." ), optparse.make_option("--results-directory", help="Location of test results"), optparse.make_option( "--skip-failing-tests", action="store_true", default=False, help= ("Skip tests that are expected to fail. Note: When using this option, " "you might miss new crashes in these tests.")), optparse.make_option("--smoke", action="store_true", help="Run just the SmokeTests"), optparse.make_option("--no-smoke", dest="smoke", action="store_false", help="Do not run just the SmokeTests"), ])) option_group_definitions.append(( "Testing Options", [ optparse.make_option( "--additional-env-var", type="string", action="append", default=[], help=("Passes that environment variable to the tests " "(--additional-env-var=NAME=VALUE)")), optparse.make_option( "--batch-size", type="int", default=None, help= ("Run a the tests in batches (n), after every n tests, the driver is " "relaunched.")), optparse.make_option( "--build", dest="build", action="store_true", default=True, help=("Check to ensure the build is up to date (default).")), optparse.make_option( "--no-build", dest="build", action="store_false", help="Don't check to see if the build is up to date."), optparse.make_option("--child-processes", help="Number of drivers to run in parallel."), optparse.make_option( "--enable-wptserve", dest="enable_wptserve", action="store_true", default=False, help= "Enable running web-platform-tests using WPTserve instead of Apache." ), optparse.make_option( "--disable-breakpad", action="store_true", help="Don't use breakpad to symbolize unexpected crashes."), optparse.make_option( "--driver-logging", action="store_true", help="Print detailed logging of the driver/content_shell"), optparse.make_option( "--enable-leak-detection", action="store_true", help="Enable the leak detection of DOM objects."), optparse.make_option( "--enable-sanitizer", action="store_true", help="Only alert on sanitizer-related errors and crashes"), optparse.make_option( "--exit-after-n-crashes-or-timeouts", type="int", default=None, help= "Exit after the first N crashes instead of running all tests"), optparse.make_option( "--exit-after-n-failures", type="int", default=None, help= "Exit after the first N failures instead of running all tests" ), optparse.make_option( "--ignore-builder-category", action="store", help= ("The category of builders to use with the --ignore-flaky-tests option " "('layout' or 'deps').")), optparse.make_option( "--ignore-flaky-tests", action="store", help= ("Control whether tests that are flaky on the bots get ignored. " "'very-flaky' == Ignore any tests that flaked more than once on the bot. " "'maybe-flaky' == Ignore any tests that flaked once on the bot. " "'unexpected' == Ignore any tests that had unexpected results on the bot." )), optparse.make_option( "--iterations", type="int", default=1, help="Number of times to run the set of tests (e.g. ABCABCABC)" ), optparse.make_option( "--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards"), optparse.make_option( "--nocheck-sys-deps", action="store_true", default=False, help="Don't check the system dependencies (themes)"), optparse.make_option( "--order", action="store", default="natural", help= ("determine the order in which the test cases will be run. " "'none' == use the order in which the tests were listed " "either in arguments or test list, " "'natural' == use the natural order (default), " "'random-seeded' == randomize the test order using a fixed seed, " "'random' == randomize the test order.")), optparse.make_option("--profile", action="store_true", help="Output per-test profile information."), optparse.make_option( "--profiler", action="store", help= "Output per-test profile information, using the specified profiler." ), optparse.make_option( "--repeat-each", type="int", default=1, help="Number of times to run each test (e.g. AAABBBCCC)"), # TODO(joelo): Delete --retry-failures and --no-retry-failures as they # are redundant with --num-retries. optparse.make_option( "--retry-failures", action="store_true", help= ("Re-try any tests that produce unexpected results. Default is to not retry " "if an explicit list of tests is passed to run-webkit-tests." )), optparse.make_option( "--no-retry-failures", dest="retry_failures", action="store_false", help="Don't re-try any tests that produce unexpected results." ), optparse.make_option( "--num-retries", type="int", default=3, help= ("Number of times to retry failures, default is 3. Only relevant when " "failure retries are enabled.")), optparse.make_option( "--run-chunk", help= "Run a specified chunk (n:l), the nth of len l, of the layout tests" ), optparse.make_option( "--run-part", help= "Run a specified part (n:m), the nth of m parts, of the layout tests" ), optparse.make_option( "--run-singly", action="store_true", default=False, help="DEPRECATED, same as --batch-size=1 --verbose"), optparse.make_option( "--skipped", action="store", default=None, help= ("control how tests marked SKIP are run. " "'default' == Skip tests unless explicitly listed on the command line, " "'ignore' == Run them anyway, " "'only' == only run the SKIP tests, " "'always' == always skip, even if listed on the command line." )), optparse.make_option( "--fastest", action="store", type="float", help= "Run the N% fastest tests as well as any tests listed on the command line" ), optparse.make_option("--test-list", action="append", metavar="FILE", help="read list of tests to run from file"), optparse.make_option("--time-out-ms", help="Set the timeout for each test"), optparse.make_option( "--wrapper", help= ("wrapper command to insert before invocations of the driver; option " "is split on whitespace before running. (Example: --wrapper='valgrind " "--smc-check=all')")), # FIXME: Display default number of child processes that will run. optparse.make_option("-f", "--fully-parallel", action="store_true", help="run all tests in parallel"), optparse.make_option( "-i", "--ignore-tests", action="append", default=[], help= "directories or test to ignore (may specify multiple times)"), optparse.make_option( "-n", "--dry-run", action="store_true", default=False, help= "Do everything but actually run the tests or upload results."), ])) # FIXME: Move these into json_results_generator.py. option_group_definitions.append(("Result JSON Options", [ optparse.make_option( "--build-name", default="DUMMY_BUILD_NAME", help="The name of the builder used in its path, e.g. webkit-rel."), optparse.make_option( "--step-name", default="webkit_tests", help="The name of the step in a build running this script."), optparse.make_option( "--build-number", default="DUMMY_BUILD_NUMBER", help="The build number of the builder running this script."), optparse.make_option( "--builder-name", default="", help= ("The name of the builder shown on the waterfall running this script " "e.g. WebKit.")), optparse.make_option("--master-name", help="The name of the buildbot master."), optparse.make_option( "--test-results-server", default="", help= "If specified, upload results json files to this appengine server." ), optparse.make_option( "--write-full-results-to", help= ("If specified, copy full_results.json from the results dir to the " "specified path.")), ])) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)
def parse_args(args): option_group_definitions = [] option_group_definitions.append(('Platform options', platform_options())) option_group_definitions.append( ('Configuration options', configuration_options())) option_group_definitions.append( ('Printing Options', printing.print_options())) option_group_definitions.append(( 'Android-specific Options', [ optparse.make_option( '--adb-device', action='append', default=[], dest='adb_devices', help='Run Android layout tests on these devices.'), # FIXME: Flip this to be off by default once we can log the # device setup more cleanly. optparse.make_option( '--no-android-logging', dest='android_logging', action='store_false', default=True, help= ('Do not log android-specific debug messages (default is to log as part ' 'of --debug-rwt-logging')), ])) option_group_definitions.append(( 'Results Options', [ optparse.make_option( '--add-platform-exceptions', action='store_true', default=False, help= ('Save generated results into the *most-specific-platform* directory rather ' 'than the *generic-platform* directory')), optparse.make_option( '--additional-driver-flag', '--additional-drt-flag', dest='additional_driver_flag', action='append', default=[], help= ('Additional command line flag to pass to the driver. Specify multiple ' 'times to add multiple flags.')), optparse.make_option( '--additional-expectations', action='append', default=[], help= ('Path to a test_expectations file that will override previous ' 'expectations. Specify multiple times for multiple sets of overrides.' )), optparse.make_option( '--additional-platform-directory', action='append', default=[], help= ('Additional directory where to look for test baselines (will take ' 'precedence over platform baselines). Specify multiple times to add ' 'multiple search path entries.')), optparse.make_option( '--build-directory', help= ('Path to the directory under which build files are kept (should not ' 'include configuration)')), optparse.make_option( '--clobber-old-results', action='store_true', default=False, help='Clobbers test results from previous runs.'), optparse.make_option( '--compare-port', action='store', default=None, help="Use the specified port's baselines first"), optparse.make_option('--driver-name', type='string', help='Alternative driver binary to use'), optparse.make_option( '--full-results-html', action='store_true', default=False, help= 'Show all failures in results.html, rather than only regressions' ), optparse.make_option( '--json-test-results', action='store', help='Path to write the JSON test results to.'), optparse.make_option( '--new-baseline', action='store_true', default=False, help= ('Save generated results as new baselines into the *most-specific-platform* ' "directory, overwriting whatever's already there. Equivalent to " '--reset-results --add-platform-exceptions')), optparse.make_option( '--new-test-results', action='store_true', default=False, help='Create new baselines when no expected results exist'), optparse.make_option( '--no-show-results', dest='show_results', action='store_false', default=True, help= "Don't launch a browser with results after the tests are done" ), optparse.make_option( '-p', '--pixel', '--pixel-tests', dest='pixel_tests', action='store_true', default=True, help= 'Enable pixel-to-pixel PNG comparisons (enabled by default)'), optparse.make_option( '--no-pixel', '--no-pixel-tests', dest='pixel_tests', action='store_false', default=True, help='Disable pixel-to-pixel PNG comparisons'), # FIXME: we should support a comma separated list with # --pixel-test-directory as well. optparse.make_option( '--pixel-test-directory', dest='pixel_test_directories', action='append', default=[], help= ('A directory where it is allowed to execute tests as pixel tests. Specify ' 'multiple times to add multiple directories. This option implies ' '--pixel-tests. If specified, only those tests will be executed as pixel ' 'tests that are located in one of the' ' directories enumerated with the ' 'option. Some ports may ignore this option while others can have a default ' 'value that can be overridden here.')), optparse.make_option( '--reset-results', action='store_true', default=False, help= 'Reset expectations to the generated results in their existing location.' ), optparse.make_option('--results-directory', help='Location of test results'), optparse.make_option( '--skip-failing-tests', action='store_true', default=False, help= ('Skip tests that are expected to fail. Note: When using this option, ' 'you might miss new crashes in these tests.')), optparse.make_option('--smoke', action='store_true', help='Run just the SmokeTests'), optparse.make_option('--no-smoke', dest='smoke', action='store_false', help='Do not run just the SmokeTests'), ])) option_group_definitions.append(( 'Testing Options', [ optparse.make_option( '--additional-env-var', type='string', action='append', default=[], help=('Passes that environment variable to the tests ' '(--additional-env-var=NAME=VALUE)')), optparse.make_option( '--batch-size', type='int', default=None, help= ('Run a the tests in batches (n), after every n tests, the driver is ' 'relaunched.')), optparse.make_option( '--build', dest='build', action='store_true', default=True, help=('Check to ensure the build is up to date (default).')), optparse.make_option( '--no-build', dest='build', action='store_false', help="Don't check to see if the build is up to date."), optparse.make_option('--child-processes', help='Number of drivers to run in parallel.'), optparse.make_option( '--disable-breakpad', action='store_true', help="Don't use breakpad to symbolize unexpected crashes."), optparse.make_option( '--driver-logging', action='store_true', help='Print detailed logging of the driver/content_shell'), optparse.make_option( '--enable-leak-detection', action='store_true', help='Enable the leak detection of DOM objects.'), optparse.make_option( '--enable-sanitizer', action='store_true', help='Only alert on sanitizer-related errors and crashes'), optparse.make_option( '--exit-after-n-crashes-or-timeouts', type='int', default=None, help= 'Exit after the first N crashes instead of running all tests'), optparse.make_option( '--exit-after-n-failures', type='int', default=None, help= 'Exit after the first N failures instead of running all tests' ), optparse.make_option( '--ignore-builder-category', action='store', help= ('The category of builders to use with the --ignore-flaky-tests option ' "('layout' or 'deps').")), optparse.make_option( '--ignore-flaky-tests', action='store', help= ('Control whether tests that are flaky on the bots get ignored. ' "'very-flaky' == Ignore any tests that flaked more than once on the bot. " "'maybe-flaky' == Ignore any tests that flaked once on the bot. " "'unexpected' == Ignore any tests that had unexpected results on the bot." )), optparse.make_option( '--iterations', type='int', default=1, help='Number of times to run the set of tests (e.g. ABCABCABC)' ), optparse.make_option( '--layout-tests-directory', help=('Path to a custom layout tests directory')), optparse.make_option( '--max-locked-shards', type='int', default=0, help='Set the maximum number of locked shards'), optparse.make_option( '--nocheck-sys-deps', action='store_true', default=False, help="Don't check the system dependencies (themes)"), optparse.make_option( '--order', action='store', default='random', help= ('Determine the order in which the test cases will be run. ' "'none' == use the order in which the tests were listed " 'either in arguments or test list, ' "'random' == pseudo-random order (default). Seed can be specified " 'via --seed, otherwise it will default to the current unix timestamp. ' "'natural' == use the natural order")), optparse.make_option('--profile', action='store_true', help='Output per-test profile information.'), optparse.make_option( '--profiler', action='store', help= 'Output per-test profile information, using the specified profiler.' ), optparse.make_option( '--repeat-each', type='int', default=1, help='Number of times to run each test (e.g. AAABBBCCC)'), # TODO(joelo): Delete --retry-failures and --no-retry-failures as they # are redundant with --num-retries. optparse.make_option( '--retry-failures', action='store_true', help= ('Re-try any tests that produce unexpected results. Default is to not retry ' 'if an explicit list of tests is passed to run-webkit-tests.' )), optparse.make_option( '--no-retry-failures', dest='retry_failures', action='store_false', help="Don't re-try any tests that produce unexpected results." ), optparse.make_option( '--num-retries', type='int', default=3, help= ('Number of times to retry failures, default is 3. Only relevant when ' 'failure retries are enabled.')), optparse.make_option( '--total-shards', type=int, help=('Total number of shards being used for this test run. ' 'Must be used with --shard-index. ' '(The user of this script is responsible for spawning ' 'all of the shards.)')), optparse.make_option( '--shard-index', type=int, help=('Shard index [0..total_shards) of this test run. ' 'Must be used with --total-shards.')), optparse.make_option( '--run-singly', action='store_true', default=False, help='DEPRECATED, same as --batch-size=1 --verbose'), optparse.make_option( '--seed', type='int', help=('Seed to use for random test order (default: %default). ' 'Only applicable in combination with --order=random.')), optparse.make_option( '--skipped', action='store', default=None, help= ('control how tests marked SKIP are run. ' "'default' == Skip tests unless explicitly listed on the command line, " "'ignore' == Run them anyway, " "'only' == only run the SKIP tests, " "'always' == always skip, even if listed on the command line." )), optparse.make_option( '--fastest', action='store', type='float', help= 'Run the N% fastest tests as well as any tests listed on the command line' ), optparse.make_option('--test-list', action='append', metavar='FILE', help='read list of tests to run from file'), optparse.make_option('--time-out-ms', help='Set the timeout for each test'), optparse.make_option( '--wrapper', help= ('wrapper command to insert before invocations of the driver; option ' "is split on whitespace before running. (Example: --wrapper='valgrind " "--smc-check=all')")), # FIXME: Display default number of child processes that will run. optparse.make_option('-f', '--fully-parallel', action='store_true', help='run all tests in parallel'), optparse.make_option( '-i', '--ignore-tests', action='append', default=[], help= 'directories or test to ignore (may specify multiple times)'), optparse.make_option( '-n', '--dry-run', action='store_true', default=False, help= 'Do everything but actually run the tests or upload results.'), ])) # FIXME: Move these into json_results_generator.py. option_group_definitions.append(('Result JSON Options', [ optparse.make_option( '--build-name', default='DUMMY_BUILD_NAME', help='The name of the builder used in its path, e.g. webkit-rel.'), optparse.make_option( '--step-name', default='webkit_tests', help='The name of the step in a build running this script.'), optparse.make_option( '--build-number', default='DUMMY_BUILD_NUMBER', help='The build number of the builder running this script.'), optparse.make_option( '--builder-name', default='', help= ('The name of the builder shown on the waterfall running this script ' 'e.g. WebKit.')), optparse.make_option('--master-name', help='The name of the buildbot master.'), optparse.make_option( '--test-results-server', default='', help= 'If specified, upload results json files to this appengine server.' ), optparse.make_option( '--write-full-results-to', help= ('If specified, copy full_results.json from the results dir to the ' 'specified path.')), ])) option_parser = optparse.OptionParser() for group_name, group_options in option_group_definitions: option_group = optparse.OptionGroup(option_parser, group_name) option_group.add_options(group_options) option_parser.add_option_group(option_group) return option_parser.parse_args(args)