def _process_options(self, options):
if not options.fuzzer_random_seed:
options.fuzzer_random_seed = random_utils.random_seed()
if options.total_timeout_sec:
options.tests_count = 0
if options.combine_tests:
if options.combine_min > options.combine_max:
print ('min_group_size (%d) cannot be larger than max_group_size (%d)' %
options.min_group_size, options.max_group_size)
raise base_runner.TestRunnerError()
if options.variants != 'default':
print ('Only default testing variant is supported with numfuzz')
raise base_runner.TestRunnerError()
return True
def _process_options(self, options):
if not options.fuzzer_random_seed:
options.fuzzer_random_seed = random_utils.random_seed()
if options.total_timeout_sec:
options.tests_count = 0
if options.combine_tests:
if options.combine_min > options.combine_max:
print ('min_group_size (%d) cannot be larger than max_group_size (%d)' %
options.min_group_size, options.max_group_size)
raise base_runner.TestRunnerError()
return True
def _parse_variants(self, aliases_str):
# Use developer defaults if no variant was specified.
aliases_str = aliases_str or 'dev'
aliases = aliases_str.split(',')
user_variants = set(reduce(
list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases]))
result = [v for v in ALL_VARIANTS if v in user_variants]
if len(result) == len(user_variants):
return result
for v in user_variants:
if v not in ALL_VARIANTS:
print 'Unknown variant: %s' % v
raise base_runner.TestRunnerError()
assert False, 'Unreachable'
def _process_options(self, options):
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = shlex.split(options.extra_flags)
if options.j == 0:
options.j = multiprocessing.cpu_count()
while options.random_seed == 0:
options.random_seed = random.SystemRandom().randint(-2147483648,
2147483647)
while options.fuzzer_random_seed == 0:
options.fuzzer_random_seed = random.SystemRandom().randint(-2147483648,
2147483647)
if options.total_timeout_sec:
options.tests_count = 0
if options.combine_tests:
if options.combine_min > options.combine_max:
print ('min_group_size (%d) cannot be larger than max_group_size (%d)' %
options.min_group_size, options.max_group_size)
raise base_runner.TestRunnerError()
return True
def CheckTestMode(name, option): # pragma: no cover
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
raise base_runner.TestRunnerError()
def _process_options(self, options):
global VARIANTS
if options.sancov_dir:
self.sancov_dir = options.sancov_dir
if not os.path.exists(self.sancov_dir):
print("sancov-dir %s doesn't exist" % self.sancov_dir)
raise base_runner.TestRunnerError()
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = sum(map(shlex.split, options.extra_flags), [])
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if options.random_gc_stress:
options.extra_flags += RANDOM_GC_STRESS_FLAGS
if self.build_config.asan:
options.extra_flags.append("--invoke-weak-callbacks")
options.extra_flags.append("--omit-quit")
if options.novfp3:
options.extra_flags.append("--noenable-vfp3")
if options.no_variants: # pragma: no cover
print ("Option --no-variants is deprecated. "
"Pass --variants=default instead.")
assert not options.variants
options.variants = "default"
if options.exhaustive_variants: # pragma: no cover
# TODO(machenbach): Switch infra to --variants=exhaustive after M65.
print ("Option --exhaustive-variants is deprecated. "
"Pass --variants=exhaustive instead.")
# This is used on many bots. It includes a larger set of default
# variants.
# Other options for manipulating variants still apply afterwards.
assert not options.variants
options.variants = "exhaustive"
if options.quickcheck:
assert not options.variants
options.variants = "stress,default"
options.slow_tests = "skip"
options.pass_fail_tests = "skip"
if self.build_config.predictable:
options.variants = "default"
options.extra_flags.append("--predictable")
options.extra_flags.append("--verify_predictable")
options.extra_flags.append("--no-inline-new")
# Add predictable wrapper to command prefix.
options.command_prefix = (
[sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
options.variants = "default"
if options.j == 0:
options.j = multiprocessing.cpu_count()
if options.random_seed_stress_count <= 1 and options.random_seed == 0:
options.random_seed = self._random_seed()
# Use developer defaults if no variant was specified.
options.variants = options.variants or "dev"
if options.variants == "infra_staging":
options.variants = "exhaustive"
options.infra_staging = True
# Use staging on whitelisted masters/builders.
options.infra_staging = self._use_staging(options)
# Resolve variant aliases and dedupe.
# TODO(machenbach): Don't mutate global variable. Rather pass mutated
# version as local variable.
VARIANTS = list(set(reduce(
list.__add__,
(VARIANT_ALIASES.get(v, [v]) for v in options.variants.split(",")),
[],
)))
if not set(VARIANTS).issubset(ALL_VARIANTS):
print "All variants must be in %s" % str(ALL_VARIANTS)
raise base_runner.TestRunnerError()
def CheckTestMode(name, option): # pragma: no cover
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
raise base_runner.TestRunnerError()
CheckTestMode("slow test", options.slow_tests)
CheckTestMode("pass|fail test", options.pass_fail_tests)
if self.build_config.no_i18n:
base_runner.TEST_MAP["bot_default"].remove("intl")
base_runner.TEST_MAP["default"].remove("intl")
def _process_options(self, options):
if options.sancov_dir:
self.sancov_dir = options.sancov_dir
if not os.path.exists(self.sancov_dir):
print("sancov-dir %s doesn't exist" % self.sancov_dir)
raise base_runner.TestRunnerError()
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if options.random_gc_stress:
options.extra_flags += RANDOM_GC_STRESS_FLAGS
if self.build_config.asan:
options.extra_flags.append("--invoke-weak-callbacks")
options.extra_flags.append("--omit-quit")
if options.novfp3:
options.extra_flags.append("--noenable-vfp3")
if options.no_variants: # pragma: no cover
print(
"Option --no-variants is deprecated. "
"Pass --variants=default instead.")
assert not options.variants
options.variants = "default"
if options.exhaustive_variants: # pragma: no cover
# TODO(machenbach): Switch infra to --variants=exhaustive after M65.
print(
"Option --exhaustive-variants is deprecated. "
"Pass --variants=exhaustive instead.")
# This is used on many bots. It includes a larger set of default
# variants.
# Other options for manipulating variants still apply afterwards.
assert not options.variants
options.variants = "exhaustive"
if options.quickcheck:
assert not options.variants
options.variants = "stress,default"
options.slow_tests = "skip"
options.pass_fail_tests = "skip"
if self.build_config.predictable:
options.variants = "default"
options.extra_flags.append("--predictable")
options.extra_flags.append("--verify_predictable")
options.extra_flags.append("--no-inline-new")
# Add predictable wrapper to command prefix.
options.command_prefix = ([sys.executable, PREDICTABLE_WRAPPER] +
options.command_prefix)
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
options.variants = "default"
if options.variants == "infra_staging":
options.variants = "exhaustive"
self._variants = self._parse_variants(options.variants)
def CheckTestMode(name, option): # pragma: no cover
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
raise base_runner.TestRunnerError()
CheckTestMode("slow test", options.slow_tests)
CheckTestMode("pass|fail test", options.pass_fail_tests)
if self.build_config.no_i18n:
base_runner.TEST_MAP["bot_default"].remove("intl")
base_runner.TEST_MAP["default"].remove("intl")
def _process_options(self, options):
global VARIANTS
# Special processing of other options, sorted alphabetically.
if options.buildbot:
options.network = False
if options.command_prefix and options.network:
print(
"Specifying --command-prefix disables network distribution, "
"running tests locally.")
options.network = False
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = sum(map(shlex.split, options.extra_flags), [])
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if self.build_config.asan:
options.extra_flags.append("--invoke-weak-callbacks")
options.extra_flags.append("--omit-quit")
if options.novfp3:
options.extra_flags.append("--noenable-vfp3")
if options.exhaustive_variants:
# This is used on many bots. It includes a larger set of default
# variants.
# Other options for manipulating variants still apply afterwards.
VARIANTS = EXHAUSTIVE_VARIANTS
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
VARIANTS = ["default"]
if options.j == 0:
options.j = multiprocessing.cpu_count()
if options.random_seed_stress_count <= 1 and options.random_seed == 0:
options.random_seed = self._random_seed()
def excl(*args):
"""Returns true if zero or one of multiple arguments are true."""
return reduce(lambda x, y: x + y, args) <= 1
if not excl(options.no_variants, bool(options.variants)):
print("Use only one of --no-variants or --variants.")
raise base_runner.TestRunnerError()
if options.quickcheck:
VARIANTS = ["default", "stress"]
options.slow_tests = "skip"
options.pass_fail_tests = "skip"
if options.no_variants:
VARIANTS = ["default"]
if options.variants:
VARIANTS = options.variants.split(",")
# Resolve variant aliases.
VARIANTS = reduce(
list.__add__,
(VARIANT_ALIASES.get(v, [v]) for v in VARIANTS),
[],
)
if not set(VARIANTS).issubset(ALL_VARIANTS):
print "All variants must be in %s" % str(ALL_VARIANTS)
raise base_runner.TestRunnerError()
if self.build_config.predictable:
VARIANTS = ["default"]
options.extra_flags.append("--predictable")
options.extra_flags.append("--verify_predictable")
options.extra_flags.append("--no-inline-new")
# Dedupe.
VARIANTS = list(set(VARIANTS))
if options.valgrind:
run_valgrind = os.path.join("tools", "run-valgrind.py")
# This is OK for distributed running, so we don't need to disable
# network.
options.command_prefix = (["python", "-u", run_valgrind] +
options.command_prefix)
def CheckTestMode(name, option):
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
raise base_runner.TestRunnerError()
CheckTestMode("slow test", options.slow_tests)
CheckTestMode("pass|fail test", options.pass_fail_tests)
if self.build_config.no_i18n:
base_runner.TEST_MAP["bot_default"].remove("intl")
base_runner.TEST_MAP["default"].remove("intl")
def CheckTestMode(name, option): # pragma: no cover
if option not in ['run', 'skip', 'dontcare']:
print('Unknown %s mode %s' % (name, option))
raise base_runner.TestRunnerError()
def _process_options(self, options):
if options.sancov_dir:
self.sancov_dir = options.sancov_dir
if not os.path.exists(self.sancov_dir):
print('sancov-dir %s doesn\'t exist' % self.sancov_dir)
raise base_runner.TestRunnerError()
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if options.random_gc_stress:
options.extra_flags += RANDOM_GC_STRESS_FLAGS
if self.build_config.asan:
options.extra_flags.append('--invoke-weak-callbacks')
options.extra_flags.append('--omit-quit')
if self.build_config.no_snap:
# Speed up slow nosnap runs. Allocation verification is covered by
# running mksnapshot on other builders.
options.extra_flags.append('--no-turbo-verify-allocation')
if options.novfp3:
options.extra_flags.append('--noenable-vfp3')
if options.no_variants: # pragma: no cover
print('Option --no-variants is deprecated. '
'Pass --variants=default instead.')
assert not options.variants
options.variants = 'default'
if options.exhaustive_variants: # pragma: no cover
# TODO(machenbach): Switch infra to --variants=exhaustive after M65.
print('Option --exhaustive-variants is deprecated. '
'Pass --variants=exhaustive instead.')
# This is used on many bots. It includes a larger set of default
# variants.
# Other options for manipulating variants still apply afterwards.
assert not options.variants
options.variants = 'exhaustive'
if options.quickcheck:
assert not options.variants
options.variants = 'stress,default'
options.slow_tests = 'skip'
options.pass_fail_tests = 'skip'
if self.build_config.predictable:
options.variants = 'default'
options.extra_flags.append('--predictable')
options.extra_flags.append('--verify-predictable')
options.extra_flags.append('--no-inline-new')
# Add predictable wrapper to command prefix.
options.command_prefix = ([sys.executable, PREDICTABLE_WRAPPER] +
options.command_prefix)
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
options.variants = 'default'
if options.variants == 'infra_staging':
options.variants = 'exhaustive'
self._variants = self._parse_variants(options.variants)
def CheckTestMode(name, option): # pragma: no cover
if option not in ['run', 'skip', 'dontcare']:
print('Unknown %s mode %s' % (name, option))
raise base_runner.TestRunnerError()
CheckTestMode('slow test', options.slow_tests)
CheckTestMode('pass|fail test', options.pass_fail_tests)
if self.build_config.no_i18n:
base_runner.TEST_MAP['bot_default'].remove('intl')
base_runner.TEST_MAP['default'].remove('intl')
def _process_options(self, options):
if options.sancov_dir:
self.sancov_dir = options.sancov_dir
if not os.path.exists(self.sancov_dir):
print('sancov-dir %s doesn\'t exist' % self.sancov_dir)
raise base_runner.TestRunnerError()
if options.gc_stress:
options.extra_flags += GC_STRESS_FLAGS
if options.random_gc_stress:
options.extra_flags += RANDOM_GC_STRESS_FLAGS
if self.build_config.asan:
options.extra_flags.append('--invoke-weak-callbacks')
if options.novfp3:
options.extra_flags.append('--noenable-vfp3')
if options.no_variants: # pragma: no cover
print ('Option --no-variants is deprecated. '
'Pass --variants=default instead.')
assert not options.variants
options.variants = 'default'
if options.exhaustive_variants: # pragma: no cover
# TODO(machenbach): Switch infra to --variants=exhaustive after M65.
print ('Option --exhaustive-variants is deprecated. '
'Pass --variants=exhaustive instead.')
# This is used on many bots. It includes a larger set of default
# variants.
# Other options for manipulating variants still apply afterwards.
assert not options.variants
options.variants = 'exhaustive'
if options.quickcheck:
assert not options.variants
options.variants = 'stress,default'
options.slow_tests = 'skip'
options.pass_fail_tests = 'skip'
if self.build_config.predictable:
options.variants = 'default'
options.extra_flags.append('--predictable')
options.extra_flags.append('--verify-predictable')
options.extra_flags.append('--no-inline-new')
# Add predictable wrapper to command prefix.
options.command_prefix = (
[sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)
# TODO(machenbach): Figure out how to test a bigger subset of variants on
# msan.
if self.build_config.msan:
options.variants = 'default'
if options.variants == 'infra_staging':
options.variants = 'exhaustive'
self._variants = self._parse_variants(options.variants)
def CheckTestMode(name, option): # pragma: no cover
if option not in ['run', 'skip', 'dontcare']:
print('Unknown %s mode %s' % (name, option))
raise base_runner.TestRunnerError()
CheckTestMode('slow test', options.slow_tests)
CheckTestMode('pass|fail test', options.pass_fail_tests)
if self.build_config.no_i18n:
base_runner.TEST_MAP['bot_default'].remove('intl')
base_runner.TEST_MAP['default'].remove('intl')
# TODO(machenbach): uncomment after infra side lands.
# base_runner.TEST_MAP['d8_default'].remove('intl')
if options.time and not options.json_test_results:
# We retrieve the slowest tests from the JSON output file, so create
# a temporary output file (which will automatically get deleted on exit)
# if the user didn't specify one.
self._temporary_json_output_file = tempfile.NamedTemporaryFile(
prefix="v8-test-runner-")
options.json_test_results = self._temporary_json_output_file.name
