def get_printer(self, args=None):
args = args or []
printing_options = printing.print_options()
option_parser = optparse.OptionParser(option_list=printing_options)
options, args = option_parser.parse_args(args)
host = MockHost()
self._port = host.port_factory.get('test', options)
regular_output = StringIO.StringIO()
printer = printing.Printer(host, options, regular_output)
return printer, regular_output
def parse_args(args):
option_group_definitions = []
option_group_definitions.append(('Platform options', platform_options()))
option_group_definitions.append(
('Configuration options', configuration_options()))
option_group_definitions.append(
('Printing Options', printing.print_options()))
option_group_definitions.append(
('web-platform-tests (WPT) Options', wpt_options()))
option_group_definitions.append((
'Android-specific Options',
[
optparse.make_option(
'--adb-device',
action='append',
default=[],
dest='adb_devices',
help='Run Android web tests on these devices'),
# FIXME: Flip this to be off by default once we can log the
# device setup more cleanly.
optparse.make_option(
'--no-android-logging',
dest='android_logging',
action='store_false',
default=True,
help=('Do not log android-specific debug messages (default '
'is to log as part of --debug-rwt-logging)')),
]))
option_group_definitions.append(('Fuchsia-specific Options', [
optparse.make_option(
'--zircon-logging',
dest='zircon_logging',
action='store_true',
default=True,
help=('Log Zircon debug messages (enabled by default).')),
optparse.make_option('--no-zircon-logging',
dest='zircon_logging',
action='store_false',
default=True,
help=('Do not log Zircon debug messages.')),
]))
option_group_definitions.append((
'Results Options',
[
optparse.make_option(
'--additional-driver-flag',
'--additional-drt-flag',
dest='additional_driver_flag',
action='append',
default=[],
help=
('Additional command line flag to pass to the driver. Specify multiple '
'times to add multiple flags.')),
optparse.make_option(
'--additional-expectations',
action='append',
default=[],
help=
('Path to a test_expectations file that will override previous '
'expectations. Specify multiple times for multiple sets of overrides.'
)),
optparse.make_option(
'--ignore-default-expectations',
action='store_true',
help=(
'Do not use the default set of TestExpectations files.')),
optparse.make_option(
'--additional-platform-directory',
action='append',
default=[],
help=
('Additional directory where to look for test baselines (will take '
'precedence over platform baselines). Specify multiple times to add '
'multiple search path entries.')),
optparse.make_option(
'--build-directory',
default='out',
help=
('Path to the directory where build files are kept, not including '
'configuration. In general this will be "out".')),
optparse.make_option(
'--clobber-old-results',
action='store_true',
default=False,
help='Clobbers test results from previous runs.'),
optparse.make_option(
'--compare-port',
action='store',
default=None,
help="Use the specified port's baselines first"),
optparse.make_option(
'--copy-baselines',
action='store_true',
default=False,
help=
('If the actual result is different from the current baseline, '
'copy the current baseline into the *most-specific-platform* '
'directory, or the flag-specific generic-platform directory if '
'--additional-driver-flag is specified. See --reset-results.'
)),
optparse.make_option('--driver-name',
type='string',
help='Alternative driver binary to use'),
optparse.make_option(
'--json-test-results', # New name from json_results_generator
'--write-full-results-to', # Old argument name
'--isolated-script-test-output', # Isolated API
help='Path to write the JSON test results for *all* tests.'),
# FIXME(tansell): Remove this option if nobody is found who needs it.
optparse.make_option(
'--json-failing-test-results',
help=
'Path to write the JSON test results for only *failing* tests.'
),
optparse.make_option(
'--no-show-results',
dest='show_results',
action='store_false',
default=True,
help=
"Don't launch a browser with results after the tests are done"
),
optparse.make_option(
'--reset-results',
action='store_true',
default=False,
help=
('Reset baselines to the generated results in their existing location or the default '
'location if no baseline exists. For virtual tests, reset the virtual baselines. '
'If --additional-driver-flag is specified, reset the flag-specific baselines. '
'If --copy-baselines is specified, the copied baselines will be reset.'
)),
optparse.make_option('--results-directory',
help='Location of test results'),
optparse.make_option('--smoke',
action='store_true',
help='Run just the SmokeTests'),
optparse.make_option('--no-smoke',
dest='smoke',
action='store_false',
help='Do not run just the SmokeTests'),
]))
option_group_definitions.append((
'Testing Options',
[
optparse.make_option(
'--additional-env-var',
type='string',
action='append',
default=[],
help=('Passes that environment variable to the tests '
'(--additional-env-var=NAME=VALUE)')),
optparse.make_option(
'--build',
dest='build',
action='store_true',
default=True,
help=('Check to ensure the build is up to date (default).')),
optparse.make_option(
'--no-build',
dest='build',
action='store_false',
help="Don't check to see if the build is up to date."),
optparse.make_option('--child-processes',
'--jobs',
'-j',
help='Number of drivers to run in parallel.'),
optparse.make_option(
'--disable-breakpad',
action='store_true',
help="Don't use breakpad to symbolize unexpected crashes."),
optparse.make_option(
'--driver-logging',
action='store_true',
help='Print detailed logging of the driver/content_shell'),
optparse.make_option(
'--enable-leak-detection',
action='store_true',
help='Enable the leak detection of DOM objects.'),
optparse.make_option(
'--enable-sanitizer',
action='store_true',
help='Only alert on sanitizer-related errors and crashes'),
optparse.make_option(
'--exit-after-n-crashes-or-timeouts',
type='int',
default=None,
help=
'Exit after the first N crashes instead of running all tests'),
optparse.make_option(
'--exit-after-n-failures',
type='int',
default=None,
help=
'Exit after the first N failures instead of running all tests'
),
optparse.make_option(
'--fuzzy-diff',
action='store_true',
default=False,
help=
('When running tests on an actual GPU, variance in pixel '
'output can leads image differences causing failed expectations. '
'Instead a fuzzy diff is used to account for this variance. '
'See tools/imagediff/image_diff.cc')),
optparse.make_option(
'--ignore-builder-category',
action='store',
help=
('The category of builders to use with the --ignore-flaky-tests option '
"('layout' or 'deps').")),
optparse.make_option(
'--ignore-flaky-tests',
action='store',
help=
('Control whether tests that are flaky on the bots get ignored. '
"'very-flaky' == Ignore any tests that flaked more than once on the bot. "
"'maybe-flaky' == Ignore any tests that flaked once on the bot. "
"'unexpected' == Ignore any tests that had unexpected results on the bot."
)),
optparse.make_option(
'--iterations',
'--isolated-script-test-repeat',
# TODO(crbug.com/893235): Remove the gtest alias when FindIt no longer uses it.
'--gtest_repeat',
type='int',
default=1,
help='Number of times to run the set of tests (e.g. ABCABCABC)'
),
optparse.make_option(
'--layout-tests-directory',
help=('Path to a custom web tests directory')),
optparse.make_option(
'--max-locked-shards',
type='int',
default=0,
help='Set the maximum number of locked shards'),
optparse.make_option(
'--nocheck-sys-deps',
action='store_true',
default=False,
help="Don't check the system dependencies (themes)"),
optparse.make_option(
'--order',
action='store',
default='random',
help=
('Determine the order in which the test cases will be run. '
"'none' == use the order in which the tests were listed "
'either in arguments or test list, '
"'random' == pseudo-random order (default). Seed can be specified "
'via --seed, otherwise it will default to the current unix timestamp. '
"'natural' == use the natural order")),
optparse.make_option('--profile',
action='store_true',
help='Output per-test profile information.'),
optparse.make_option(
'--profiler',
action='store',
help=
'Output per-test profile information, using the specified profiler.'
),
optparse.make_option(
'--restart-shell-between-tests',
type='choice',
action='store',
choices=[
'always',
'never',
'on_retry',
],
default='on_retry',
help=
('Restarting the shell between tests produces more '
'consistent results, as it prevents state from carrying over '
'from previous tests. It also increases test run time by at '
'least 2X. By default, the shell is restarted when tests get '
'retried, since leaking state between retries can sometimes '
'mask underlying flakiness, and the whole point of retries is '
'to look for flakiness.')),
optparse.make_option(
'--repeat-each',
type='int',
default=1,
help='Number of times to run each test (e.g. AAABBBCCC)'),
optparse.make_option(
'--num-retries',
'--test-launcher-retry-limit',
'--isolated-script-test-launcher-retry-limit',
type='int',
default=None,
help=('Number of times to retry failures. Default (when this '
'flag is not specified) is to retry 3 times, unless an '
'explicit list of tests is passed to run_web_tests.py. '
'If a non-zero value is given explicitly, failures are '
'retried regardless.')),
optparse.make_option(
'--no-retry-failures',
dest='num_retries',
action='store_const',
const=0,
help="Don't retry any failures (equivalent to --num-retries=0)."
),
optparse.make_option(
'--total-shards',
type=int,
help=('Total number of shards being used for this test run. '
'Must be used with --shard-index. '
'(The user of this script is responsible for spawning '
'all of the shards.)')),
optparse.make_option(
'--shard-index',
type=int,
help=('Shard index [0..total_shards) of this test run. '
'Must be used with --total-shards.')),
optparse.make_option(
'--seed',
type='int',
help=('Seed to use for random test order (default: %default). '
'Only applicable in combination with --order=random.')),
optparse.make_option(
'--skipped',
action='store',
default=None,
help=
('Control how tests marked SKIP are run. '
'"default" == Skip tests unless explicitly listed on the command line, '
'"ignore" == Run them anyway, '
'"only" == only run the SKIP tests, '
'"always" == always skip, even if listed on the command line.'
)),
optparse.make_option(
'--isolated-script-test-also-run-disabled-tests',
# TODO(crbug.com/893235): Remove the gtest alias when FindIt no longer uses it.
'--gtest_also_run_disabled_tests',
action='store_const',
const='ignore',
dest='skipped',
help=('Equivalent to --skipped=ignore.')),
optparse.make_option(
'--skip-failing-tests',
action='store_true',
default=False,
help=
('Skip tests that are expected to fail. Note: When using this option, '
'you might miss new crashes in these tests.')),
optparse.make_option(
'--skip-timeouts',
action='store_true',
default=False,
help=
('Skip tests marked TIMEOUT. Use it to speed up running the entire '
'test suite.')),
optparse.make_option(
'--fastest',
action='store',
type='float',
help=
'Run the N% fastest tests as well as any tests listed on the command line'
),
optparse.make_option('--test-list',
action='append',
metavar='FILE',
help='read list of tests to run from file'),
optparse.make_option(
'--isolated-script-test-filter',
type='string',
help=
'A list of tests to run separated by TWO colons, e.g. fast::css/test.html, '
'same as listing them as positional arguments'),
# TODO(crbug.com/893235): Remove gtest_filter when FindIt no longer uses it.
optparse.make_option(
'--gtest_filter',
type='string',
help='A colon-separated list of tests to run. Wildcards are '
'NOT supported. It is the same as listing the tests as '
'positional arguments.'),
optparse.make_option('--time-out-ms',
help='Set the timeout for each test'),
optparse.make_option(
'--wrapper',
help=
('wrapper command to insert before invocations of the driver; option '
'is split on whitespace before running. (Example: --wrapper="valgrind '
'--smc-check=all")')),
# FIXME: Display the default number of child processes that will run.
optparse.make_option('-f',
'--fully-parallel',
action='store_true',
help='run all tests in parallel'),
optparse.make_option(
'-i',
'--ignore-tests',
action='append',
default=[],
help=
'directories or test to ignore (may specify multiple times)'),
optparse.make_option(
'-n',
'--dry-run',
action='store_true',
default=False,
help=
'Do everything but actually run the tests or upload results.'),
optparse.make_option(
'-w',
'--watch',
action='store_true',
help='Re-run tests quickly (e.g. avoid restarting the server)'
),
optparse.make_option(
'--zero-tests-executed-ok',
action='store_true',
help='If set, exit with a success code when no tests are run.'
' Used on trybots when web tests are retried without patch.')
]))
# FIXME: Move these into json_results_generator.py.
option_group_definitions.append(('Result JSON Options', [
optparse.make_option(
'--build-name',
default='DUMMY_BUILD_NAME',
help='The name of the builder used in its path, e.g. webkit-rel.'),
optparse.make_option(
'--step-name',
default='webkit_tests',
help='The name of the step in a build running this script.'),
optparse.make_option(
'--build-number',
default='DUMMY_BUILD_NUMBER',
help='The build number of the builder running this script.'),
optparse.make_option(
'--builder-name',
default='',
help=
('The name of the builder shown on the waterfall running this script '
'e.g. WebKit.')),
optparse.make_option('--master-name',
help='The name of the buildbot master.'),
optparse.make_option(
'--test-results-server',
default='',
help=
'If specified, upload results json files to this appengine server.'
),
]))
option_parser = optparse.OptionParser(
prog='run_web_tests.py',
usage='%prog [options] [tests]',
description=
'Runs Blink web tests as described in docs/testing/web_tests.md')
for group_name, group_options in option_group_definitions:
option_group = optparse.OptionGroup(option_parser, group_name)
option_group.add_options(group_options)
option_parser.add_option_group(option_group)
(options, args) = option_parser.parse_args(args)
return (options, args)
def get_options(args):
print_options = printing.print_options()
option_parser = optparse.OptionParser(option_list=print_options)
return option_parser.parse_args(args)
