def _load_build_config(self, options): for outdir in self._possible_outdirs(options): try: self.build_config = self._do_load_build_config( outdir, options.verbose) except TestRunnerError: pass if not self.build_config: # pragma: no cover print('Failed to load build config') raise TestRunnerError print('Build found: %s' % self.outdir) if str(self.build_config): print('>>> Autodetected:') print(self.build_config) # Represents the OS where tests are run on. Same as host OS except for # Android, which is determined by build output. if self.build_config.is_android: self.target_os = 'android' else: self.target_os = utils.GuessOS()
def _Run(self, runnable, count, secondary=False): suffix = ' - secondary' if secondary else '' shell_dir = self.shell_dir_secondary if secondary else self.shell_dir title = ">>> %%s (#%d)%s:" % ((count + 1), suffix) cmd = runnable.GetCommand(self.command_prefix, shell_dir, self.extra_flags) try: output = cmd.execute() except OSError: # pragma: no cover logging.exception(title % "OSError") raise logging.info(title % "Stdout" + "\n%s", output.stdout) if output.stderr: # pragma: no cover # Print stderr for debugging. logging.info(title % "Stderr" + "\n%s", output.stderr) if output.timed_out: logging.warning(">>> Test timed out after %ss.", runnable.timeout) raise TestFailedError() if output.exit_code != 0: logging.warning(">>> Test crashed.") raise TestFailedError() if '--prof' in self.extra_flags: os_prefix = {"linux": "linux", "macos": "mac"}.get(utils.GuessOS()) if os_prefix: tick_tools = os.path.join(TOOLS_BASE, "%s-tick-processor" % os_prefix) subprocess.check_call(tick_tools + " --only-summary", shell=True) else: # pragma: no cover logging.warning( "Profiler option currently supported on Linux and Mac OS.") # time outputs to stderr if runnable.process_size: return output.stdout + output.stderr return output.stdout
def Execute(arch, mode, args, options, suites): print(">>> Running tests for %s.%s" % (arch, mode)) shell_dir = options.shell_dir if not shell_dir: if options.auto_detect: # If an output dir with a build was passed, test directly in that # directory. shell_dir = os.path.join(BASE_DIR, options.outdir) elif options.buildbot: # TODO(machenbach): Get rid of different output folder location on # buildbot. Currently this is capitalized Release and Debug. shell_dir = os.path.join(BASE_DIR, options.outdir, mode) mode = BuildbotToV8Mode(mode) else: shell_dir = os.path.join( BASE_DIR, options.outdir, "%s.%s" % (arch, MODES[mode]["output_folder"]), ) if not os.path.exists(shell_dir): raise Exception('Could not find shell_dir: "%s"' % shell_dir) # Populate context object. mode_flags = MODES[mode]["flags"] # Simulators are slow, therefore allow a longer timeout. if arch in SLOW_ARCHS: options.timeout *= 2 options.timeout *= MODES[mode]["timeout_scalefactor"] if options.predictable: # Predictable mode is slower. options.timeout *= 2 ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir, mode_flags, options.verbose, options.timeout, options.isolates, options.command_prefix, options.extra_flags, options.no_i18n, options.random_seed, options.no_sorting, options.rerun_failures_count, options.rerun_failures_max, options.predictable, options.no_harness, use_perf_data=not options.swarming, sancov_dir=options.sancov_dir) # TODO(all): Combine "simulator" and "simulator_run". # TODO(machenbach): In GN we can derive simulator run from # target_arch != v8_target_arch in the dumped build config. simulator_run = not options.dont_skip_simulator_slow_tests and \ arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \ 'ppc', 'ppc64'] and \ ARCH_GUESS and arch != ARCH_GUESS # Find available test suites and read test cases from them. variables = { "arch": arch, "asan": options.asan, "deopt_fuzzer": False, "gc_stress": options.gc_stress, "gcov_coverage": options.gcov_coverage, "isolates": options.isolates, "mode": MODES[mode]["status_mode"], "no_i18n": options.no_i18n, "no_snap": options.no_snap, "simulator_run": simulator_run, "simulator": utils.UseSimulator(arch), "system": utils.GuessOS(), "tsan": options.tsan, "msan": options.msan, "dcheck_always_on": options.dcheck_always_on, "novfp3": options.novfp3, "predictable": options.predictable, "byteorder": sys.byteorder, } all_tests = [] num_tests = 0 for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) all_tests += s.tests # First filtering by status applying the generic rules (independent of # variants). s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, options.pass_fail_tests) if options.cat: verbose.PrintTestSource(s.tests) continue variant_gen = s.CreateVariantGenerator(VARIANTS) variant_tests = [ t.CopyAddingFlags(v, flags) for t in s.tests for v in variant_gen.FilterVariantsByTest(t) for flags in variant_gen.GetFlagSets(t, v) ] if options.random_seed_stress_count > 1: # Duplicate test for random seed stress mode. def iter_seed_flags(): for i in range(0, options.random_seed_stress_count): # Use given random seed for all runs (set by default in execution.py) # or a new random seed if none is specified. if options.random_seed: yield [] else: yield ["--random-seed=%d" % RandomSeed()] s.tests = [ t.CopyAddingFlags(t.variant, flags) for t in variant_tests for flags in iter_seed_flags() ] else: s.tests = variant_tests # Second filtering by status applying the variant-dependent rules. s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, options.pass_fail_tests, variants=True) s.tests = ShardTests(s.tests, options) num_tests += len(s.tests) if options.cat: return 0 # We're done here. if options.report: verbose.PrintReport(all_tests) # Run the tests, either locally or distributed on the network. start_time = time.time() progress_indicator = progress.IndicatorNotifier() progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]()) if options.junitout: progress_indicator.Register(progress.JUnitTestProgressIndicator( options.junitout, options.junittestsuite)) if options.json_test_results: progress_indicator.Register(progress.JsonTestProgressIndicator( options.json_test_results, arch, MODES[mode]["execution_mode"], ctx.random_seed)) if options.flakiness_results: progress_indicator.Register(progress.FlakinessTestProgressIndicator( options.flakiness_results)) run_networked = not options.no_network if not run_networked: if options.verbose: print("Network distribution disabled, running tests locally.") elif utils.GuessOS() != "linux": print("Network distribution is only supported on Linux, sorry!") run_networked = False peers = [] if run_networked: peers = network_execution.GetPeers() if not peers: print("No connection to distribution server; running tests locally.") run_networked = False elif len(peers) == 1: print("No other peers on the network; running tests locally.") run_networked = False elif num_tests <= 100: print("Less than 100 tests, running them locally.") run_networked = False if run_networked: runner = network_execution.NetworkedRunner(suites, progress_indicator, ctx, peers, BASE_DIR) else: runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) overall_duration = time.time() - start_time if options.time: verbose.PrintTestDurations(suites, overall_duration) if num_tests == 0: print("Warning: no tests were run!") if exit_code == 1 and options.json_test_results: print("Force exit code 0 after failures. Json test results file generated " "with failure information.") exit_code = 0 if options.sancov_dir: # If tests ran with sanitizer coverage, merge coverage files in the end. try: print "Merging sancov files." subprocess.check_call([ sys.executable, join(BASE_DIR, "tools", "sanitizers", "sancov_merger.py"), "--coverage-dir=%s" % options.sancov_dir]) except: print >> sys.stderr, "Error: Merging sancov files failed." exit_code = 1 return exit_code
def BuildOptions(): result = optparse.OptionParser() result.usage = '%prog [options] [tests]' result.description = """TESTS: %s""" % (TEST_MAP["default"]) result.add_option("--arch", help=("The architecture to run tests for, " "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS), default="ia32,x64,arm") result.add_option("--arch-and-mode", help="Architecture and mode in the format 'arch.mode'", default=None) result.add_option("--asan", help="Regard test expectations for ASAN", default=False, action="store_true") result.add_option("--sancov-dir", help="Directory where to collect coverage data") result.add_option("--cfi-vptr", help="Run tests with UBSAN cfi_vptr option.", default=False, action="store_true") result.add_option("--buildbot", help="Adapt to path structure used on buildbots", default=False, action="store_true") result.add_option("--dcheck-always-on", help="Indicates that V8 was compiled with DCHECKs enabled", default=False, action="store_true") result.add_option("--novfp3", help="Indicates that V8 was compiled without VFP3 support", default=False, action="store_true") result.add_option("--cat", help="Print the source of the tests", default=False, action="store_true") result.add_option("--slow-tests", help="Regard slow tests (run|skip|dontcare)", default="dontcare") result.add_option("--pass-fail-tests", help="Regard pass|fail tests (run|skip|dontcare)", default="dontcare") result.add_option("--gc-stress", help="Switch on GC stress mode", default=False, action="store_true") result.add_option("--gcov-coverage", help="Uses executables instrumented for gcov coverage", default=False, action="store_true") result.add_option("--command-prefix", help="Prepended to each shell command used to run a test", default="") result.add_option("--download-data", help="Download missing test suite data", default=False, action="store_true") result.add_option("--download-data-only", help="Deprecated", default=False, action="store_true") result.add_option("--extra-flags", help="Additional flags to pass to each test command", default="") result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") result.add_option("-j", help="The number of parallel tasks to run", default=0, type="int") result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated," " uppercase for ninja and buildbot builds): %s" % MODES.keys(), default="release,debug") result.add_option("--no-harness", "--noharness", help="Run without test harness of a given suite", default=False, action="store_true") result.add_option("--no-i18n", "--noi18n", help="Skip internationalization tests", default=False, action="store_true") result.add_option("--no-network", "--nonetwork", help="Don't distribute tests on the network", default=(utils.GuessOS() != "linux"), dest="no_network", action="store_true") result.add_option("--no-presubmit", "--nopresubmit", help='Skip presubmit checks (deprecated)', default=False, dest="no_presubmit", action="store_true") result.add_option("--no-snap", "--nosnap", help='Test a build compiled without snapshot.', default=False, dest="no_snap", action="store_true") result.add_option("--no-sorting", "--nosorting", help="Don't sort tests according to duration of last run.", default=False, dest="no_sorting", action="store_true") result.add_option("--no-variants", "--novariants", help="Don't run any testing variants", default=False, dest="no_variants", action="store_true") result.add_option("--variants", help="Comma-separated list of testing variants;" " default: \"%s\"" % ",".join(VARIANTS)) result.add_option("--exhaustive-variants", default=False, action="store_true", help="Use exhaustive set of default variants:" " \"%s\"" % ",".join(EXHAUSTIVE_VARIANTS)) result.add_option("--outdir", help="Base directory with compile output", default="out") result.add_option("--gn", help="Scan out.gn for the last built configuration", default=False, action="store_true") result.add_option("--predictable", help="Compare output of several reruns of each test", default=False, action="store_true") result.add_option("-p", "--progress", help=("The style of progress indicator" " (verbose, dots, color, mono)"), choices=progress.PROGRESS_INDICATORS.keys(), default="mono") result.add_option("--quickcheck", default=False, action="store_true", help=("Quick check mode (skip slow tests)")) result.add_option("--report", help="Print a summary of the tests to be run", default=False, action="store_true") result.add_option("--json-test-results", help="Path to a file for storing json results.") result.add_option("--flakiness-results", help="Path to a file for storing flakiness json.") result.add_option("--rerun-failures-count", help=("Number of times to rerun each failing test case. " "Very slow tests will be rerun only once."), default=0, type="int") result.add_option("--rerun-failures-max", help="Maximum number of failing test cases to rerun.", default=100, type="int") result.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") result.add_option("--shard-run", help="Run this shard from the split up tests.", default=1, type="int") result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") result.add_option("--shell-dir", help="Directory containing executables", default="") result.add_option("--dont-skip-slow-simulator-tests", help="Don't skip more slow tests when using a simulator.", default=False, action="store_true", dest="dont_skip_simulator_slow_tests") result.add_option("--swarming", help="Indicates running test driver on swarming.", default=False, action="store_true") result.add_option("--time", help="Print timing information after running", default=False, action="store_true") result.add_option("-t", "--timeout", help="Timeout in seconds", default=TIMEOUT_DEFAULT, type="int") result.add_option("--tsan", help="Regard test expectations for TSAN", default=False, action="store_true") result.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") result.add_option("--valgrind", help="Run tests through valgrind", default=False, action="store_true") result.add_option("--warn-unused", help="Report unused rules", default=False, action="store_true") result.add_option("--junitout", help="File name of the JUnit output") result.add_option("--junittestsuite", help="The testsuite name in the JUnit output file", default="v8tests") result.add_option("--random-seed", default=0, dest="random_seed", type="int", help="Default seed for initializing random generator") result.add_option("--random-seed-stress-count", default=1, type="int", dest="random_seed_stress_count", help="Number of runs with different random seeds") result.add_option("--msan", help="Regard test expectations for MSAN", default=False, action="store_true") return result
def _execute(self, args, options, suites): print(">>> Running tests for %s.%s" % (self.build_config.arch, self.mode_name)) # Populate context object. # Simulators are slow, therefore allow a longer timeout. if self.build_config.arch in SLOW_ARCHS: options.timeout *= 2 options.timeout *= self.mode_options.timeout_scalefactor if self.build_config.predictable: # Predictable mode is slower. options.timeout *= 2 ctx = context.Context(self.build_config.arch, self.mode_options.execution_mode, self.outdir, self.mode_options.flags, options.verbose, options.timeout, options.isolates, options.command_prefix, options.extra_flags, self.build_config.no_i18n, options.random_seed, options.no_sorting, options.rerun_failures_count, options.rerun_failures_max, options.no_harness, use_perf_data=not options.swarming, sancov_dir=self.sancov_dir, infra_staging=options.infra_staging) # TODO(all): Combine "simulator" and "simulator_run". # TODO(machenbach): In GN we can derive simulator run from # target_arch != v8_target_arch in the dumped build config. simulator_run = ( not options.dont_skip_simulator_slow_tests and self.build_config.arch in [ 'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc', 'ppc64', 's390', 's390x'] and bool(base_runner.ARCH_GUESS) and self.build_config.arch != base_runner.ARCH_GUESS) # Find available test suites and read test cases from them. variables = { "arch": self.build_config.arch, "asan": self.build_config.asan, "byteorder": sys.byteorder, "dcheck_always_on": self.build_config.dcheck_always_on, "deopt_fuzzer": False, "gc_fuzzer": False, "gc_stress": options.gc_stress or options.random_gc_stress, "gcov_coverage": self.build_config.gcov_coverage, "isolates": options.isolates, "mode": self.mode_options.status_mode, "msan": self.build_config.msan, "no_harness": options.no_harness, "no_i18n": self.build_config.no_i18n, "no_snap": self.build_config.no_snap, "novfp3": options.novfp3, "predictable": self.build_config.predictable, "simulator": utils.UseSimulator(self.build_config.arch), "simulator_run": simulator_run, "system": utils.GuessOS(), "tsan": self.build_config.tsan, "ubsan_vptr": self.build_config.ubsan_vptr, } progress_indicator = progress.IndicatorNotifier() progress_indicator.Register( progress.PROGRESS_INDICATORS[options.progress]()) if options.junitout: # pragma: no cover progress_indicator.Register(progress.JUnitTestProgressIndicator( options.junitout, options.junittestsuite)) if options.json_test_results: progress_indicator.Register(progress.JsonTestProgressIndicator( options.json_test_results, self.build_config.arch, self.mode_options.execution_mode, ctx.random_seed)) if options.flakiness_results: # pragma: no cover progress_indicator.Register(progress.FlakinessTestProgressIndicator( options.flakiness_results)) if options.infra_staging: for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) return self._run_test_procs(suites, args, options, progress_indicator, ctx) all_tests = [] num_tests = 0 for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) all_tests += s.tests # First filtering by status applying the generic rules (tests without # variants) if options.warn_unused: tests = [(t.name, t.variant) for t in s.tests] s.statusfile.warn_unused_rules(tests, check_variant_rules=False) s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests) if options.cat: verbose.PrintTestSource(s.tests) continue variant_gen = s.CreateLegacyVariantsGenerator(VARIANTS) variant_tests = [ t.create_variant(v, flags) for t in s.tests for v in variant_gen.FilterVariantsByTest(t) for flags in variant_gen.GetFlagSets(t, v) ] if options.random_seed_stress_count > 1: # Duplicate test for random seed stress mode. def iter_seed_flags(): for _ in range(0, options.random_seed_stress_count): # Use given random seed for all runs (set by default in # execution.py) or a new random seed if none is specified. if options.random_seed: yield [] else: yield ["--random-seed=%d" % self._random_seed()] s.tests = [ t.create_variant(t.variant, flags, 'seed-stress-%d' % n) for t in variant_tests for n, flags in enumerate(iter_seed_flags()) ] else: s.tests = variant_tests # Second filtering by status applying also the variant-dependent rules. if options.warn_unused: tests = [(t.name, t.variant) for t in s.tests] s.statusfile.warn_unused_rules(tests, check_variant_rules=True) s.FilterTestCasesByStatus(options.slow_tests, options.pass_fail_tests) s.tests = self._shard_tests(s.tests, options) for t in s.tests: t.cmd = t.get_command(ctx) num_tests += len(s.tests) if options.cat: return 0 # We're done here. if options.report: verbose.PrintReport(all_tests) # Run the tests. start_time = time.time() if self.build_config.predictable: outproc_factory = predictable.get_outproc else: outproc_factory = None runner = execution.Runner(suites, progress_indicator, ctx, outproc_factory) exit_code = runner.Run(options.j) overall_duration = time.time() - start_time if options.time: verbose.PrintTestDurations(suites, runner.outputs, overall_duration) if num_tests == 0: exit_code = 3 print("Warning: no tests were run!") if exit_code == 1 and options.json_test_results: print("Force exit code 0 after failures. Json test results file " "generated with failure information.") exit_code = 0 if self.sancov_dir: # If tests ran with sanitizer coverage, merge coverage files in the end. try: print "Merging sancov files." subprocess.check_call([ sys.executable, join(self.basedir, "tools", "sanitizers", "sancov_merger.py"), "--coverage-dir=%s" % self.sancov_dir]) except: print >> sys.stderr, "Error: Merging sancov files failed." exit_code = 1 return exit_code
def Execute(arch, mode, args, options, suites, workspace): print(">>> Running tests for %s.%s" % (arch, mode)) shell_dir = options.shell_dir if not shell_dir: if options.buildbot: # TODO(machenbach): Get rid of different output folder location on # buildbot. Currently this is capitalized Release and Debug. shell_dir = os.path.join(workspace, options.outdir, mode) mode = mode.lower() else: shell_dir = os.path.join( workspace, options.outdir, "%s.%s" % (arch, MODES[mode]["output_folder"]), ) shell_dir = os.path.relpath(shell_dir) # Populate context object. mode_flags = MODES[mode]["flags"] timeout = options.timeout if timeout == -1: # Simulators are slow, therefore allow a longer default timeout. if arch in SLOW_ARCHS: timeout = 2 * TIMEOUT_DEFAULT else: timeout = TIMEOUT_DEFAULT timeout *= MODES[mode]["timeout_scalefactor"] if options.predictable: # Predictable mode is slower. timeout *= 2 # TODO(machenbach): Remove temporary verbose output on windows after # debugging driver-hung-up on XP. verbose_output = (options.verbose or utils.IsWindows() and options.progress == "verbose") ctx = context.Context( arch, MODES[mode]["execution_mode"], shell_dir, mode_flags, verbose_output, timeout, options.isolates, options.command_prefix, options.extra_flags, options.no_i18n, options.random_seed, options.no_sorting, options.rerun_failures_count, options.rerun_failures_max, options.predictable, options.no_harness) # TODO(all): Combine "simulator" and "simulator_run". simulator_run = not options.dont_skip_simulator_slow_tests and \ arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64el', \ 'ppc', 'ppc64'] and \ ARCH_GUESS and arch != ARCH_GUESS # Find available test suites and read test cases from them. variables = { "arch": arch, "asan": options.asan, "deopt_fuzzer": False, "gc_stress": options.gc_stress, "isolates": options.isolates, "mode": MODES[mode]["status_mode"], "no_i18n": options.no_i18n, "no_snap": options.no_snap, "simulator_run": simulator_run, "simulator": utils.UseSimulator(arch), "system": utils.GuessOS(), "tsan": options.tsan, "msan": options.msan, "dcheck_always_on": options.dcheck_always_on, "byteorder": sys.byteorder, } all_tests = [] num_tests = 0 test_id = 0 for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) all_tests += s.tests s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests, options.slow_tests, options.pass_fail_tests) if options.cat: verbose.PrintTestSource(s.tests) continue variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS] s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in s.VariantFlags(t, variant_flags) ] s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 if options.cat: return 0 # We're done here. if options.report: verbose.PrintReport(all_tests) if num_tests == 0: print "No tests to run." return 0 # Run the tests, either locally or distributed on the network. start_time = time.time() progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() if options.junitout: progress_indicator = progress.JUnitTestProgressIndicator( progress_indicator, options.junitout, options.junittestsuite) if options.json_test_results: progress_indicator = progress.JsonTestProgressIndicator( progress_indicator, options.json_test_results, arch, MODES[mode]["execution_mode"]) run_networked = not options.no_network if not run_networked: print("Network distribution disabled, running tests locally.") elif utils.GuessOS() != "linux": print("Network distribution is only supported on Linux, sorry!") run_networked = False peers = [] if run_networked: peers = network_execution.GetPeers() if not peers: print( "No connection to distribution server; running tests locally.") run_networked = False elif len(peers) == 1: print("No other peers on the network; running tests locally.") run_networked = False elif num_tests <= 100: print("Less than 100 tests, running them locally.") run_networked = False if run_networked: runner = network_execution.NetworkedRunner(suites, progress_indicator, ctx, peers, workspace) else: runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) overall_duration = time.time() - start_time if options.time: verbose.PrintTestDurations(suites, overall_duration) return exit_code
def BuildOptions(): result = optparse.OptionParser() result.add_option("--arch", help=("The architecture to run tests for, " "'auto' or 'native' for auto-detect"), default="ia32,x64,arm") result.add_option("--arch-and-mode", help="Architecture and mode in the format 'arch.mode'", default=None) result.add_option("--asan", help="Regard test expectations for ASAN", default=False, action="store_true") result.add_option("--buildbot", help="Adapt to path structure used on buildbots", default=False, action="store_true") result.add_option( "--dcheck-always-on", help="Indicates that V8 was compiled with DCHECKs enabled", default=False, action="store_true") result.add_option("--cat", help="Print the source of the tests", default=False, action="store_true") result.add_option("--flaky-tests", help="Regard tests marked as flaky (run|skip|dontcare)", default="dontcare") result.add_option("--slow-tests", help="Regard slow tests (run|skip|dontcare)", default="dontcare") result.add_option("--pass-fail-tests", help="Regard pass|fail tests (run|skip|dontcare)", default="dontcare") result.add_option("--gc-stress", help="Switch on GC stress mode", default=False, action="store_true") result.add_option( "--command-prefix", help="Prepended to each shell command used to run a test", default="") result.add_option("--download-data", help="Download missing test suite data", default=False, action="store_true") result.add_option("--download-data-only", help="Download missing test suite data and exit", default=False, action="store_true") result.add_option("--extra-flags", help="Additional flags to pass to each test command", default="") result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") result.add_option("-j", help="The number of parallel tasks to run", default=0, type="int") result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)", default="release,debug") result.add_option("--no-harness", "--noharness", help="Run without test harness of a given suite", default=False, action="store_true") result.add_option("--no-i18n", "--noi18n", help="Skip internationalization tests", default=False, action="store_true") result.add_option("--no-network", "--nonetwork", help="Don't distribute tests on the network", default=(utils.GuessOS() != "linux"), dest="no_network", action="store_true") result.add_option("--no-presubmit", "--nopresubmit", help='Skip presubmit checks', default=False, dest="no_presubmit", action="store_true") result.add_option("--no-snap", "--nosnap", help='Test a build compiled without snapshot.', default=False, dest="no_snap", action="store_true") result.add_option( "--no-sorting", "--nosorting", help="Don't sort tests according to duration of last run.", default=False, dest="no_sorting", action="store_true") result.add_option( "--no-stress", "--nostress", help="Don't run crankshaft --always-opt --stress-op test", default=False, dest="no_stress", action="store_true") result.add_option("--no-variants", "--novariants", help="Don't run any testing variants", default=False, dest="no_variants", action="store_true") result.add_option("--variants", help="Comma-separated list of testing variants") result.add_option("--outdir", help="Base directory with compile output", default="out") result.add_option("--predictable", help="Compare output of several reruns of each test", default=False, action="store_true") result.add_option("-p", "--progress", help=("The style of progress indicator" " (verbose, dots, color, mono)"), choices=progress.PROGRESS_INDICATORS.keys(), default="mono") result.add_option("--quickcheck", default=False, action="store_true", help=("Quick check mode (skip slow/flaky tests)")) result.add_option("--report", help="Print a summary of the tests to be run", default=False, action="store_true") result.add_option("--json-test-results", help="Path to a file for storing json results.") result.add_option("--rerun-failures-count", help=("Number of times to rerun each failing test case. " "Very slow tests will be rerun only once."), default=0, type="int") result.add_option("--rerun-failures-max", help="Maximum number of failing test cases to rerun.", default=100, type="int") result.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") result.add_option("--shard-run", help="Run this shard from the split up tests.", default=1, type="int") result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") result.add_option("--shell-dir", help="Directory containing executables", default="") result.add_option( "--dont-skip-slow-simulator-tests", help="Don't skip more slow tests when using a simulator.", default=False, action="store_true", dest="dont_skip_simulator_slow_tests") result.add_option("--stress-only", help="Only run tests with --always-opt --stress-opt", default=False, action="store_true") result.add_option("--time", help="Print timing information after running", default=False, action="store_true") result.add_option("-t", "--timeout", help="Timeout in seconds", default=-1, type="int") result.add_option("--tsan", help="Regard test expectations for TSAN", default=False, action="store_true") result.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") result.add_option("--valgrind", help="Run tests through valgrind", default=False, action="store_true") result.add_option("--warn-unused", help="Report unused rules", default=False, action="store_true") result.add_option("--junitout", help="File name of the JUnit output") result.add_option("--junittestsuite", help="The testsuite name in the JUnit output file", default="v8tests") result.add_option("--random-seed", default=0, dest="random_seed", help="Default seed for initializing random generator") result.add_option("--msan", help="Regard test expectations for MSAN", default=False, action="store_true") return result
def Execute(arch, mode, args, options, suites, workspace): print(">>> Running tests for %s.%s" % (arch, mode)) dist = Distribution(options) shell_dir = options.shell_dir if not shell_dir: if options.buildbot: shell_dir = os.path.join(workspace, options.outdir, mode) mode = mode.lower() else: shell_dir = os.path.join(workspace, options.outdir, "%s.%s" % (arch, mode)) shell_dir = os.path.relpath(shell_dir) # Populate context object. mode_flags = MODE_FLAGS[mode] timeout = options.timeout if timeout == -1: # Simulators are slow, therefore allow a longer default timeout. if arch in SLOW_ARCHS: timeout = 2 * TIMEOUT_DEFAULT else: timeout = TIMEOUT_DEFAULT timeout *= TIMEOUT_SCALEFACTOR[mode] ctx = context.Context(arch, mode, shell_dir, mode_flags, options.verbose, timeout, options.isolates, options.command_prefix, options.extra_flags, False) # Find available test suites and read test cases from them. variables = { "arch": arch, "asan": options.asan, "deopt_fuzzer": True, "gc_stress": False, "isolates": options.isolates, "mode": mode, "no_i18n": False, "simulator": utils.UseSimulator(arch), "system": utils.GuessOS(), } all_tests = [] num_tests = 0 test_id = 0 # Remember test case prototypes for the fuzzing phase. test_backup = dict((s, []) for s in suites) for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) all_tests += s.tests s.FilterTestCasesByStatus(False) test_backup[s] = s.tests analysis_flags = [ "--deopt-every-n-times", "%d" % MAX_DEOPT, "--print-deopt-stress" ] s.tests = [t.CopyAddingFlags(analysis_flags) for t in s.tests] num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 if num_tests == 0: print "No tests to run." return 0 try: print(">>> Collection phase") progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) if runner.terminate: return exit_code except KeyboardInterrupt: return 1 print(">>> Analysis phase") num_tests = 0 test_id = 0 for s in suites: test_results = {} for t in s.tests: for line in t.output.stdout.splitlines(): if line.startswith("=== Stress deopt counter: "): test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1]) for t in s.tests: if t.path not in test_results: print "Missing results for %s" % t.path if options.dump_results_file: results_dict = dict( (t.path, n) for (t, n) in test_results.iteritems()) with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f: f.write(json.dumps(results_dict)) # Reset tests and redistribute the prototypes from the collection phase. s.tests = [] if options.verbose: print "Test distributions:" for t in test_backup[s]: max_deopt = test_results.get(t.path, 0) if max_deopt == 0: continue n_deopt = CalculateNTests(max_deopt, options) distribution = dist.Distribute(n_deopt, max_deopt) if options.verbose: print "%s %s" % (t.path, distribution) for i in distribution: fuzzing_flags = ["--deopt-every-n-times", "%d" % i] s.tests.append(t.CopyAddingFlags(fuzzing_flags)) num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 if num_tests == 0: print "No tests to run." return 0 try: print(">>> Deopt fuzzing phase (%d test cases)" % num_tests) progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) if runner.terminate: return exit_code except KeyboardInterrupt: return 1 return exit_code
def BuildOptions(): result = optparse.OptionParser() result.add_option("--arch", help=("The architecture to run tests for, " "'auto' or 'native' for auto-detect"), default="ia32,x64,arm") result.add_option("--arch-and-mode", help="Architecture and mode in the format 'arch.mode'", default=None) result.add_option("--buildbot", help="Adapt to path structure used on buildbots", default=False, action="store_true") result.add_option("--cat", help="Print the source of the tests", default=False, action="store_true") result.add_option("--command-prefix", help="Prepended to each shell command used to run a test", default="") result.add_option("--download-data", help="Download missing test suite data", default=False, action="store_true") result.add_option("--extra-flags", help="Additional flags to pass to each test command", default="") result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") result.add_option("-j", help="The number of parallel tasks to run", default=0, type="int") result.add_option("-m", "--mode", help="The test modes in which to run (comma-separated)", default="release,debug") result.add_option("--no-network", "--nonetwork", help="Don't distribute tests on the network", default=(utils.GuessOS() != "linux"), dest="no_network", action="store_true") result.add_option("--no-presubmit", "--nopresubmit", help='Skip presubmit checks', default=False, dest="no_presubmit", action="store_true") result.add_option("--no-stress", "--nostress", help="Don't run crankshaft --always-opt --stress-op test", default=False, dest="no_stress", action="store_true") result.add_option("--outdir", help="Base directory with compile output", default="out") result.add_option("-p", "--progress", help=("The style of progress indicator" " (verbose, dots, color, mono)"), choices=progress.PROGRESS_INDICATORS.keys(), default="mono") result.add_option("--report", help="Print a summary of the tests to be run", default=False, action="store_true") result.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") result.add_option("--shard-run", help="Run this shard from the split up tests.", default=1, type="int") result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") result.add_option("--shell-dir", help="Directory containing executables", default="") result.add_option("--stress-only", help="Only run tests with --always-opt --stress-opt", default=False, action="store_true") result.add_option("--time", help="Print timing information after running", default=False, action="store_true") result.add_option("-t", "--timeout", help="Timeout in seconds", default= -1, type="int") result.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") result.add_option("--valgrind", help="Run tests through valgrind", default=False, action="store_true") result.add_option("--warn-unused", help="Report unused rules", default=False, action="store_true") return result
def Execute(arch, mode, args, options, suites, workspace): print(">>> Running tests for %s.%s" % (arch, mode)) shell_dir = options.shell_dir if not shell_dir: if options.buildbot: shell_dir = os.path.join(workspace, options.outdir, mode) mode = mode.lower() else: shell_dir = os.path.join(workspace, options.outdir, "%s.%s" % (arch, mode)) shell_dir = os.path.relpath(shell_dir) # Populate context object. mode_flags = MODE_FLAGS[mode] options.timeout *= TIMEOUT_SCALEFACTOR[mode] ctx = context.Context(arch, mode, shell_dir, mode_flags, options.verbose, options.timeout, options.isolates, options.command_prefix, options.extra_flags) # Find available test suites and read test cases from them. variables = { "mode": mode, "arch": arch, "system": utils.GuessOS(), "isolates": options.isolates } all_tests = [] num_tests = 0 test_id = 0 for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) all_tests += s.tests if len(args) > 0: s.FilterTestCasesByArgs(args) s.FilterTestCasesByStatus(options.warn_unused) if options.cat: verbose.PrintTestSource(s.tests) continue variant_flags = s.VariantFlags() or VARIANT_FLAGS s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in variant_flags ] s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 if options.cat: return 0 # We're done here. if options.report: verbose.PrintReport(all_tests) if num_tests == 0: print "No tests to run." return 0 # Run the tests, either locally or distributed on the network. try: start_time = time.time() progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() run_networked = not options.no_network if not run_networked: print("Network distribution disabled, running tests locally.") elif utils.GuessOS() != "linux": print("Network distribution is only supported on Linux, sorry!") run_networked = False peers = [] if run_networked: peers = network_execution.GetPeers() if not peers: print("No connection to distribution server; running tests locally.") run_networked = False elif len(peers) == 1: print("No other peers on the network; running tests locally.") run_networked = False elif num_tests <= 100: print("Less than 100 tests, running them locally.") run_networked = False if run_networked: runner = network_execution.NetworkedRunner(suites, progress_indicator, ctx, peers, workspace) else: runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) if runner.terminate: return exit_code overall_duration = time.time() - start_time except KeyboardInterrupt: return 1 if options.time: verbose.PrintTestDurations(suites, overall_duration) return exit_code
""" # for py2/py3 compatibility from __future__ import absolute_import from __future__ import print_function import sys from testrunner.local import command from testrunner.local import utils MAX_TRIES = 3 TIMEOUT = 120 # Predictable mode works only when run on the host os. command.setup(utils.GuessOS(), None) def maybe_decode(message): if not isinstance(message, str): return message.decode() return message def main(args): def allocation_str(stdout): for line in reversed((stdout or '').splitlines()): if maybe_decode(line).startswith('### Allocations = '): return line return None
def _setup_env(self, options): """Setup additional environment variables.""" # Many tests assume an English interface. os.environ['LANG'] = 'en_US.UTF-8' external_symbolizer_path = os.path.join( base_runner.BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer', ) if utils.IsWindows(): # Quote, because sanitizers might confuse colon as option separator. external_symbolizer_path = '"%s.exe"' % external_symbolizer_path symbolizer = 'external_symbolizer_path=%s' % external_symbolizer_path if self.build_config.asan: asan_options = [symbolizer, "allow_user_segv_handler=1"] if not utils.GuessOS() in ['macos', 'windows']: # LSAN is not available on mac and windows. asan_options.append('detect_leaks=1') os.environ['ASAN_OPTIONS'] = ":".join(asan_options) if options.sancov_dir: assert os.path.exists(options.sancov_dir) os.environ['ASAN_OPTIONS'] = ":".join([ 'coverage=1', 'coverage_dir=%s' % options.sancov_dir, symbolizer, "allow_user_segv_handler=1", ]) if self.build_config.cfi_vptr: os.environ['UBSAN_OPTIONS'] = ":".join([ 'print_stacktrace=1', 'print_summary=1', 'symbolize=1', symbolizer, ]) if self.build_config.ubsan_vptr: os.environ['UBSAN_OPTIONS'] = ":".join([ 'print_stacktrace=1', symbolizer, ]) if self.build_config.msan: os.environ['MSAN_OPTIONS'] = symbolizer if self.build_config.tsan: suppressions_file = os.path.join(base_runner.BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt') os.environ['TSAN_OPTIONS'] = " ".join([ symbolizer, 'suppressions=%s' % suppressions_file, 'exit_code=0', 'report_thread_leaks=0', 'history_size=7', 'report_destroy_locked=0', ])
def _execute(self, args, options, suites): print(">>> Running tests for %s.%s" % (self.build_config.arch, self.mode_name)) dist = self._distribution(options) # Populate context object. timeout = options.timeout if timeout == -1: # Simulators are slow, therefore allow a longer default timeout. if self.build_config.arch in SLOW_ARCHS: timeout = 2 * TIMEOUT_DEFAULT else: timeout = TIMEOUT_DEFAULT timeout *= self.mode_options.timeout_scalefactor ctx = context.Context( self.build_config.arch, self.mode_options.execution_mode, self.outdir, self.mode_options.flags, options.verbose, timeout, options.isolates, options.command_prefix, options.extra_flags, False, # Keep i18n on by default. options.random_seed, True, # No sorting of test cases. 0, # Don't rerun failing tests. 0, # No use of a rerun-failing-tests maximum. False, # No predictable mode. False, # No no_harness mode. False, # Don't use perf data. False) # Coverage not supported. # Find available test suites and read test cases from them. variables = { "arch": self.build_config.arch, "asan": self.build_config.asan, "byteorder": sys.byteorder, "dcheck_always_on": self.build_config.dcheck_always_on, "deopt_fuzzer": True, "gc_fuzzer": False, "gc_stress": False, "gcov_coverage": self.build_config.gcov_coverage, "isolates": options.isolates, "mode": self.mode_options.status_mode, "msan": self.build_config.msan, "no_harness": False, "no_i18n": self.build_config.no_i18n, "no_snap": self.build_config.no_snap, "novfp3": False, "predictable": self.build_config.predictable, "simulator": utils.UseSimulator(self.build_config.arch), "simulator_run": False, "system": utils.GuessOS(), "tsan": self.build_config.tsan, "ubsan_vptr": self.build_config.ubsan_vptr, } num_tests = 0 test_id = 0 # Remember test case prototypes for the fuzzing phase. test_backup = dict((s, []) for s in suites) for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) s.FilterTestCasesByStatus(False) test_backup[s] = s.tests analysis_flags = [ "--deopt-every-n-times", "%d" % MAX_DEOPT, "--print-deopt-stress" ] s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ] num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 if num_tests == 0: print "No tests to run." return 0 print(">>> Collection phase") progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) print(">>> Analysis phase") num_tests = 0 test_id = 0 for s in suites: test_results = {} for t in s.tests: for line in t.output.stdout.splitlines(): if line.startswith("=== Stress deopt counter: "): test_results[t.path] = MAX_DEOPT - int( line.split(" ")[-1]) for t in s.tests: if t.path not in test_results: print "Missing results for %s" % t.path if options.dump_results_file: results_dict = dict( (t.path, n) for (t, n) in test_results.iteritems()) with file( "%s.%d.txt" % (options.dump_results_file, time.time()), "w") as f: f.write(json.dumps(results_dict)) # Reset tests and redistribute the prototypes from the collection phase. s.tests = [] if options.verbose: print "Test distributions:" for t in test_backup[s]: max_deopt = test_results.get(t.path, 0) if max_deopt == 0: continue n_deopt = self._calculate_n_tests(max_deopt, options) distribution = dist.Distribute(n_deopt, max_deopt) if options.verbose: print "%s %s" % (t.path, distribution) for i in distribution: fuzzing_flags = ["--deopt-every-n-times", "%d" % i] s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags)) num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 if num_tests == 0: print "No tests to run." return 0 print(">>> Deopt fuzzing phase (%d test cases)" % num_tests) progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() runner = execution.Runner(suites, progress_indicator, ctx) code = runner.Run(options.j) return exit_code or code
def Execute(arch, mode, args, options, suites, workspace): print(">>> Running tests for %s.%s" % (arch, mode)) shell_dir = options.shell_dir if not shell_dir: if options.buildbot: shell_dir = os.path.join(workspace, options.outdir, mode) mode = mode.lower() else: shell_dir = os.path.join(workspace, options.outdir, "%s.%s" % (arch, mode)) shell_dir = os.path.relpath(shell_dir) if mode == "optdebug": mode = "debug" # "optdebug" is just an alias. # Populate context object. mode_flags = MODE_FLAGS[mode] timeout = options.timeout if timeout == -1: # Simulators are slow, therefore allow a longer default timeout. if arch in SLOW_ARCHS: timeout = 2 * TIMEOUT_DEFAULT else: timeout = TIMEOUT_DEFAULT timeout *= TIMEOUT_SCALEFACTOR[mode] ctx = context.Context(arch, mode, shell_dir, mode_flags, options.verbose, timeout, options.isolates, options.command_prefix, options.extra_flags, options.no_i18n) # TODO(all): Combine "simulator" and "simulator_run". simulator_run = not options.dont_skip_simulator_slow_tests and \ arch in ['a64', 'arm', 'mips'] and ARCH_GUESS and arch != ARCH_GUESS # Find available test suites and read test cases from them. variables = { "arch": arch, "asan": options.asan, "deopt_fuzzer": False, "gc_stress": options.gc_stress, "isolates": options.isolates, "mode": mode, "no_i18n": options.no_i18n, "simulator_run": simulator_run, "simulator": utils.UseSimulator(arch), "system": utils.GuessOS(), } all_tests = [] num_tests = 0 test_id = 0 for s in suites: s.ReadStatusFile(variables) s.ReadTestCases(ctx) if len(args) > 0: s.FilterTestCasesByArgs(args) all_tests += s.tests s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests, options.slow_tests, options.pass_fail_tests) if options.cat: verbose.PrintTestSource(s.tests) continue variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS] s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in s.VariantFlags(t, variant_flags) ] s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) num_tests += len(s.tests) for t in s.tests: t.id = test_id test_id += 1 if options.cat: return 0 # We're done here. if options.report: verbose.PrintReport(all_tests) if num_tests == 0: print "No tests to run." return 0 # Run the tests, either locally or distributed on the network. try: start_time = time.time() progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() if options.junitout: progress_indicator = progress.JUnitTestProgressIndicator( progress_indicator, options.junitout, options.junittestsuite) run_networked = not options.no_network if not run_networked: print("Network distribution disabled, running tests locally.") elif utils.GuessOS() != "linux": print("Network distribution is only supported on Linux, sorry!") run_networked = False peers = [] if run_networked: peers = network_execution.GetPeers() if not peers: print( "No connection to distribution server; running tests locally." ) run_networked = False elif len(peers) == 1: print("No other peers on the network; running tests locally.") run_networked = False elif num_tests <= 100: print("Less than 100 tests, running them locally.") run_networked = False if run_networked: runner = network_execution.NetworkedRunner(suites, progress_indicator, ctx, peers, workspace) else: runner = execution.Runner(suites, progress_indicator, ctx) exit_code = runner.Run(options.j) if runner.terminate: return exit_code overall_duration = time.time() - start_time except KeyboardInterrupt: raise if options.time: verbose.PrintTestDurations(suites, overall_duration) return exit_code
def _do_execute(self, options, args): # Use the v8 root as cwd as some test cases use "load" with relative # paths. os.chdir(base_runner.BASE_DIR) self._setup_env(options) if options.swarming: # Swarming doesn't print how isolated commands are called. Lets make # this less cryptic by printing it ourselves. print ' '.join(sys.argv) if utils.GuessOS() == "macos": # TODO(machenbach): Temporary output for investigating hanging test # driver on mac. print "V8 related processes running on this host:" try: print subprocess.check_output( "ps -e | egrep 'd8|cctest|unittests'", shell=True) except Exception: pass exit_code = 0 suite_paths = utils.GetSuitePaths(join(base_runner.BASE_DIR, "test")) # Use default tests if no test configuration was provided at the cmd line. if len(args) == 0: args = ["default"] # Expand arguments with grouped tests. The args should reflect the list # of suites as otherwise filters would break. def ExpandTestGroups(name): if name in base_runner.TEST_MAP: return [suite for suite in base_runner.TEST_MAP[name]] else: return [name] args = reduce(lambda x, y: x + y, [ExpandTestGroups(arg) for arg in args], []) args_suites = OrderedDict() # Used as set for arg in args: args_suites[arg.split('/')[0]] = True suite_paths = [ s for s in args_suites if s in suite_paths ] suites = [] for root in suite_paths: suite = testsuite.TestSuite.LoadTestSuite( os.path.join(base_runner.BASE_DIR, "test", root)) if suite: suites.append(suite) for s in suites: s.PrepareSources() for (arch, mode) in self.arch_and_mode: try: code = self._execute(arch, mode, args, options, suites) except KeyboardInterrupt: return 2 exit_code = exit_code or code return exit_code
""" # for py2/py3 compatibility from __future__ import print_function from __future__ import absolute_import import sys from testrunner.local import command from testrunner.local import utils MAX_TRIES = 3 TIMEOUT = 120 # Predictable mode works only when run on the host os. command.setup(utils.GuessOS()) def main(args): def allocation_str(stdout): for line in reversed((stdout or '').splitlines()): if line.startswith('### Allocations = '): return line return None cmd = command.Command(args[0], args[1:], timeout=TIMEOUT) previous_allocations = None for run in range(1, MAX_TRIES + 1): print('### Predictable run #%d' % run) output = cmd.execute()