def _process_options(self, options): if not options.fuzzer_random_seed: options.fuzzer_random_seed = random_utils.random_seed() if options.total_timeout_sec: options.tests_count = 0 if options.combine_tests: if options.combine_min > options.combine_max: print ('min_group_size (%d) cannot be larger than max_group_size (%d)' % options.min_group_size, options.max_group_size) raise base_runner.TestRunnerError() if options.variants != 'default': print ('Only default testing variant is supported with numfuzz') raise base_runner.TestRunnerError() return True
def _process_options(self, options): if not options.fuzzer_random_seed: options.fuzzer_random_seed = random_utils.random_seed() if options.total_timeout_sec: options.tests_count = 0 if options.combine_tests: if options.combine_min > options.combine_max: print ('min_group_size (%d) cannot be larger than max_group_size (%d)' % options.min_group_size, options.max_group_size) raise base_runner.TestRunnerError() return True
def _parse_variants(self, aliases_str): # Use developer defaults if no variant was specified. aliases_str = aliases_str or 'dev' aliases = aliases_str.split(',') user_variants = set(reduce( list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases])) result = [v for v in ALL_VARIANTS if v in user_variants] if len(result) == len(user_variants): return result for v in user_variants: if v not in ALL_VARIANTS: print 'Unknown variant: %s' % v raise base_runner.TestRunnerError() assert False, 'Unreachable'
def _process_options(self, options): options.command_prefix = shlex.split(options.command_prefix) options.extra_flags = shlex.split(options.extra_flags) if options.j == 0: options.j = multiprocessing.cpu_count() while options.random_seed == 0: options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647) while options.fuzzer_random_seed == 0: options.fuzzer_random_seed = random.SystemRandom().randint(-2147483648, 2147483647) if options.total_timeout_sec: options.tests_count = 0 if options.combine_tests: if options.combine_min > options.combine_max: print ('min_group_size (%d) cannot be larger than max_group_size (%d)' % options.min_group_size, options.max_group_size) raise base_runner.TestRunnerError() return True
def CheckTestMode(name, option): # pragma: no cover if not option in ["run", "skip", "dontcare"]: print "Unknown %s mode %s" % (name, option) raise base_runner.TestRunnerError()
def _process_options(self, options): global VARIANTS if options.sancov_dir: self.sancov_dir = options.sancov_dir if not os.path.exists(self.sancov_dir): print("sancov-dir %s doesn't exist" % self.sancov_dir) raise base_runner.TestRunnerError() options.command_prefix = shlex.split(options.command_prefix) options.extra_flags = sum(map(shlex.split, options.extra_flags), []) if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS if options.random_gc_stress: options.extra_flags += RANDOM_GC_STRESS_FLAGS if self.build_config.asan: options.extra_flags.append("--invoke-weak-callbacks") options.extra_flags.append("--omit-quit") if options.novfp3: options.extra_flags.append("--noenable-vfp3") if options.no_variants: # pragma: no cover print ("Option --no-variants is deprecated. " "Pass --variants=default instead.") assert not options.variants options.variants = "default" if options.exhaustive_variants: # pragma: no cover # TODO(machenbach): Switch infra to --variants=exhaustive after M65. print ("Option --exhaustive-variants is deprecated. " "Pass --variants=exhaustive instead.") # This is used on many bots. It includes a larger set of default # variants. # Other options for manipulating variants still apply afterwards. assert not options.variants options.variants = "exhaustive" if options.quickcheck: assert not options.variants options.variants = "stress,default" options.slow_tests = "skip" options.pass_fail_tests = "skip" if self.build_config.predictable: options.variants = "default" options.extra_flags.append("--predictable") options.extra_flags.append("--verify_predictable") options.extra_flags.append("--no-inline-new") # Add predictable wrapper to command prefix. options.command_prefix = ( [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. if self.build_config.msan: options.variants = "default" if options.j == 0: options.j = multiprocessing.cpu_count() if options.random_seed_stress_count <= 1 and options.random_seed == 0: options.random_seed = self._random_seed() # Use developer defaults if no variant was specified. options.variants = options.variants or "dev" if options.variants == "infra_staging": options.variants = "exhaustive" options.infra_staging = True # Use staging on whitelisted masters/builders. options.infra_staging = self._use_staging(options) # Resolve variant aliases and dedupe. # TODO(machenbach): Don't mutate global variable. Rather pass mutated # version as local variable. VARIANTS = list(set(reduce( list.__add__, (VARIANT_ALIASES.get(v, [v]) for v in options.variants.split(",")), [], ))) if not set(VARIANTS).issubset(ALL_VARIANTS): print "All variants must be in %s" % str(ALL_VARIANTS) raise base_runner.TestRunnerError() def CheckTestMode(name, option): # pragma: no cover if not option in ["run", "skip", "dontcare"]: print "Unknown %s mode %s" % (name, option) raise base_runner.TestRunnerError() CheckTestMode("slow test", options.slow_tests) CheckTestMode("pass|fail test", options.pass_fail_tests) if self.build_config.no_i18n: base_runner.TEST_MAP["bot_default"].remove("intl") base_runner.TEST_MAP["default"].remove("intl")
def _process_options(self, options): if options.sancov_dir: self.sancov_dir = options.sancov_dir if not os.path.exists(self.sancov_dir): print("sancov-dir %s doesn't exist" % self.sancov_dir) raise base_runner.TestRunnerError() if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS if options.random_gc_stress: options.extra_flags += RANDOM_GC_STRESS_FLAGS if self.build_config.asan: options.extra_flags.append("--invoke-weak-callbacks") options.extra_flags.append("--omit-quit") if options.novfp3: options.extra_flags.append("--noenable-vfp3") if options.no_variants: # pragma: no cover print( "Option --no-variants is deprecated. " "Pass --variants=default instead.") assert not options.variants options.variants = "default" if options.exhaustive_variants: # pragma: no cover # TODO(machenbach): Switch infra to --variants=exhaustive after M65. print( "Option --exhaustive-variants is deprecated. " "Pass --variants=exhaustive instead.") # This is used on many bots. It includes a larger set of default # variants. # Other options for manipulating variants still apply afterwards. assert not options.variants options.variants = "exhaustive" if options.quickcheck: assert not options.variants options.variants = "stress,default" options.slow_tests = "skip" options.pass_fail_tests = "skip" if self.build_config.predictable: options.variants = "default" options.extra_flags.append("--predictable") options.extra_flags.append("--verify_predictable") options.extra_flags.append("--no-inline-new") # Add predictable wrapper to command prefix. options.command_prefix = ([sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. if self.build_config.msan: options.variants = "default" if options.variants == "infra_staging": options.variants = "exhaustive" self._variants = self._parse_variants(options.variants) def CheckTestMode(name, option): # pragma: no cover if not option in ["run", "skip", "dontcare"]: print "Unknown %s mode %s" % (name, option) raise base_runner.TestRunnerError() CheckTestMode("slow test", options.slow_tests) CheckTestMode("pass|fail test", options.pass_fail_tests) if self.build_config.no_i18n: base_runner.TEST_MAP["bot_default"].remove("intl") base_runner.TEST_MAP["default"].remove("intl")
def _process_options(self, options): global VARIANTS # Special processing of other options, sorted alphabetically. if options.buildbot: options.network = False if options.command_prefix and options.network: print( "Specifying --command-prefix disables network distribution, " "running tests locally.") options.network = False options.command_prefix = shlex.split(options.command_prefix) options.extra_flags = sum(map(shlex.split, options.extra_flags), []) if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS if self.build_config.asan: options.extra_flags.append("--invoke-weak-callbacks") options.extra_flags.append("--omit-quit") if options.novfp3: options.extra_flags.append("--noenable-vfp3") if options.exhaustive_variants: # This is used on many bots. It includes a larger set of default # variants. # Other options for manipulating variants still apply afterwards. VARIANTS = EXHAUSTIVE_VARIANTS # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. if self.build_config.msan: VARIANTS = ["default"] if options.j == 0: options.j = multiprocessing.cpu_count() if options.random_seed_stress_count <= 1 and options.random_seed == 0: options.random_seed = self._random_seed() def excl(*args): """Returns true if zero or one of multiple arguments are true.""" return reduce(lambda x, y: x + y, args) <= 1 if not excl(options.no_variants, bool(options.variants)): print("Use only one of --no-variants or --variants.") raise base_runner.TestRunnerError() if options.quickcheck: VARIANTS = ["default", "stress"] options.slow_tests = "skip" options.pass_fail_tests = "skip" if options.no_variants: VARIANTS = ["default"] if options.variants: VARIANTS = options.variants.split(",") # Resolve variant aliases. VARIANTS = reduce( list.__add__, (VARIANT_ALIASES.get(v, [v]) for v in VARIANTS), [], ) if not set(VARIANTS).issubset(ALL_VARIANTS): print "All variants must be in %s" % str(ALL_VARIANTS) raise base_runner.TestRunnerError() if self.build_config.predictable: VARIANTS = ["default"] options.extra_flags.append("--predictable") options.extra_flags.append("--verify_predictable") options.extra_flags.append("--no-inline-new") # Dedupe. VARIANTS = list(set(VARIANTS)) if options.valgrind: run_valgrind = os.path.join("tools", "run-valgrind.py") # This is OK for distributed running, so we don't need to disable # network. options.command_prefix = (["python", "-u", run_valgrind] + options.command_prefix) def CheckTestMode(name, option): if not option in ["run", "skip", "dontcare"]: print "Unknown %s mode %s" % (name, option) raise base_runner.TestRunnerError() CheckTestMode("slow test", options.slow_tests) CheckTestMode("pass|fail test", options.pass_fail_tests) if self.build_config.no_i18n: base_runner.TEST_MAP["bot_default"].remove("intl") base_runner.TEST_MAP["default"].remove("intl")
def CheckTestMode(name, option): # pragma: no cover if option not in ['run', 'skip', 'dontcare']: print('Unknown %s mode %s' % (name, option)) raise base_runner.TestRunnerError()
def _process_options(self, options): if options.sancov_dir: self.sancov_dir = options.sancov_dir if not os.path.exists(self.sancov_dir): print('sancov-dir %s doesn\'t exist' % self.sancov_dir) raise base_runner.TestRunnerError() if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS if options.random_gc_stress: options.extra_flags += RANDOM_GC_STRESS_FLAGS if self.build_config.asan: options.extra_flags.append('--invoke-weak-callbacks') options.extra_flags.append('--omit-quit') if self.build_config.no_snap: # Speed up slow nosnap runs. Allocation verification is covered by # running mksnapshot on other builders. options.extra_flags.append('--no-turbo-verify-allocation') if options.novfp3: options.extra_flags.append('--noenable-vfp3') if options.no_variants: # pragma: no cover print('Option --no-variants is deprecated. ' 'Pass --variants=default instead.') assert not options.variants options.variants = 'default' if options.exhaustive_variants: # pragma: no cover # TODO(machenbach): Switch infra to --variants=exhaustive after M65. print('Option --exhaustive-variants is deprecated. ' 'Pass --variants=exhaustive instead.') # This is used on many bots. It includes a larger set of default # variants. # Other options for manipulating variants still apply afterwards. assert not options.variants options.variants = 'exhaustive' if options.quickcheck: assert not options.variants options.variants = 'stress,default' options.slow_tests = 'skip' options.pass_fail_tests = 'skip' if self.build_config.predictable: options.variants = 'default' options.extra_flags.append('--predictable') options.extra_flags.append('--verify-predictable') options.extra_flags.append('--no-inline-new') # Add predictable wrapper to command prefix. options.command_prefix = ([sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. if self.build_config.msan: options.variants = 'default' if options.variants == 'infra_staging': options.variants = 'exhaustive' self._variants = self._parse_variants(options.variants) def CheckTestMode(name, option): # pragma: no cover if option not in ['run', 'skip', 'dontcare']: print('Unknown %s mode %s' % (name, option)) raise base_runner.TestRunnerError() CheckTestMode('slow test', options.slow_tests) CheckTestMode('pass|fail test', options.pass_fail_tests) if self.build_config.no_i18n: base_runner.TEST_MAP['bot_default'].remove('intl') base_runner.TEST_MAP['default'].remove('intl')
def _process_options(self, options): if options.sancov_dir: self.sancov_dir = options.sancov_dir if not os.path.exists(self.sancov_dir): print('sancov-dir %s doesn\'t exist' % self.sancov_dir) raise base_runner.TestRunnerError() if options.gc_stress: options.extra_flags += GC_STRESS_FLAGS if options.random_gc_stress: options.extra_flags += RANDOM_GC_STRESS_FLAGS if self.build_config.asan: options.extra_flags.append('--invoke-weak-callbacks') if options.novfp3: options.extra_flags.append('--noenable-vfp3') if options.no_variants: # pragma: no cover print ('Option --no-variants is deprecated. ' 'Pass --variants=default instead.') assert not options.variants options.variants = 'default' if options.exhaustive_variants: # pragma: no cover # TODO(machenbach): Switch infra to --variants=exhaustive after M65. print ('Option --exhaustive-variants is deprecated. ' 'Pass --variants=exhaustive instead.') # This is used on many bots. It includes a larger set of default # variants. # Other options for manipulating variants still apply afterwards. assert not options.variants options.variants = 'exhaustive' if options.quickcheck: assert not options.variants options.variants = 'stress,default' options.slow_tests = 'skip' options.pass_fail_tests = 'skip' if self.build_config.predictable: options.variants = 'default' options.extra_flags.append('--predictable') options.extra_flags.append('--verify-predictable') options.extra_flags.append('--no-inline-new') # Add predictable wrapper to command prefix. options.command_prefix = ( [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) # TODO(machenbach): Figure out how to test a bigger subset of variants on # msan. if self.build_config.msan: options.variants = 'default' if options.variants == 'infra_staging': options.variants = 'exhaustive' self._variants = self._parse_variants(options.variants) def CheckTestMode(name, option): # pragma: no cover if option not in ['run', 'skip', 'dontcare']: print('Unknown %s mode %s' % (name, option)) raise base_runner.TestRunnerError() CheckTestMode('slow test', options.slow_tests) CheckTestMode('pass|fail test', options.pass_fail_tests) if self.build_config.no_i18n: base_runner.TEST_MAP['bot_default'].remove('intl') base_runner.TEST_MAP['default'].remove('intl') # TODO(machenbach): uncomment after infra side lands. # base_runner.TEST_MAP['d8_default'].remove('intl') if options.time and not options.json_test_results: # We retrieve the slowest tests from the JSON output file, so create # a temporary output file (which will automatically get deleted on exit) # if the user didn't specify one. self._temporary_json_output_file = tempfile.NamedTemporaryFile( prefix="v8-test-runner-") options.json_test_results = self._temporary_json_output_file.name