from testrunner.local import execution from testrunner.local import progress from testrunner.local import testsuite from testrunner.local.variants import ALL_VARIANTS from testrunner.local import utils from testrunner.local import verbose from testrunner.network import network_execution from testrunner.objects import context # Base dir of the v8 checkout to be used as cwd. BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DEFAULT_OUT_GN = "out.gn" ARCH_GUESS = utils.DefaultArch() # Map of test name synonyms to lists of test suites. Should be ordered by # expected runtimes (suites with slow test cases first). These groups are # invoked in seperate steps on the bots. TEST_MAP = { # This needs to stay in sync with test/bot_default.isolate. "bot_default": [ "debugger", "mjsunit", "cctest", "inspector", "webkit", "fuzzer", "message", "preparser",
def Main(argv): parser = argparse.ArgumentParser() parser.add_argument('--arch', help='The architecture to run tests for. Pass "auto" ' 'to auto-detect.', default='x64', choices=SUPPORTED_ARCHS + ['auto']) parser.add_argument( '--buildbot', help='Adapt to path structure used on buildbots and adds ' 'timestamps/level to all logged status messages', default=False, action='store_true') parser.add_argument('-d', '--device', help='The device ID to run Android tests on. If not ' 'given it will be autodetected.') parser.add_argument('--extra-flags', help='Additional flags to pass to the test executable', default='') parser.add_argument('--json-test-results', help='Path to a file for storing json results.') parser.add_argument( '--json-test-results-secondary', help='Path to a file for storing json results from run ' 'without patch or for reference build run.') parser.add_argument('--outdir', help='Base directory with compile output', default='out') parser.add_argument( '--outdir-secondary', help='Base directory with compile output without patch ' 'or for reference build') parser.add_argument('--binary-override-path', help='JavaScript engine binary. By default, d8 under ' 'architecture-specific build dir. ' 'Not supported in conjunction with outdir-secondary.') parser.add_argument('--prioritize', help='Raise the priority to nice -20 for the ' 'benchmarking process.Requires Linux, schedtool, and ' 'sudo privileges.', default=False, action='store_true') parser.add_argument( '--affinitize', help='Run benchmarking process on the specified core. ' 'For example: --affinitize=0 will run the benchmark ' 'process on core 0. --affinitize=3 will run the ' 'benchmark process on core 3. Requires Linux, schedtool, ' 'and sudo privileges.', default=None) parser.add_argument( '--noaslr', help='Disable ASLR for the duration of the benchmarked ' 'process. Requires Linux and sudo privileges.', default=False, action='store_true') parser.add_argument( '--cpu-governor', help='Set cpu governor to specified policy for the ' 'duration of the benchmarked process. Typical options: ' '"powersave" for more stable results, or "performance" ' 'for shorter completion time of suite, with potentially ' 'more noise in results.') parser.add_argument( '--filter', help='Only run the benchmarks beginning with this ' 'string. For example: ' '--filter=JSTests/TypedArrays/ will run only TypedArray ' 'benchmarks from the JSTests suite.', default='') parser.add_argument( '--confidence-level', type=float, help='Repeatedly runs each benchmark until specified ' 'confidence level is reached. The value is interpreted ' 'as the number of standard deviations from the mean that ' 'all values must lie within. Typical values are 1, 2 and ' '3 and correspond to 68%%, 95%% and 99.7%% probability ' 'that the measured value is within 0.1%% of the true ' 'value. Larger values result in more retries and thus ' 'longer runtime, but also provide more reliable results. ' 'Also see --max-total-duration flag.') parser.add_argument( '--max-total-duration', type=int, default=7140, # 1h 59m help='Max total duration in seconds allowed for retries ' 'across all tests. This is especially useful in ' 'combination with the --confidence-level flag.') parser.add_argument( '--dump-logcats-to', help='Writes logcat output from each test into specified ' 'directory. Only supported for android targets.') parser.add_argument('--run-count', type=int, default=0, help='Override the run count specified by the test ' 'suite. The default 0 uses the suite\'s config.') parser.add_argument('-v', '--verbose', default=False, action='store_true', help='Be verbose and print debug output.') parser.add_argument('suite', nargs='+', help='Path to the suite config file.') try: args = parser.parse_args(argv) except SystemExit: return INFRA_FAILURE_RETCODE logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s') if args.arch == 'auto': # pragma: no cover args.arch = utils.DefaultArch() if args.arch not in SUPPORTED_ARCHS: logging.error('Auto-detected architecture "%s" is not supported.', args.arch) return INFRA_FAILURE_RETCODE if (args.json_test_results_secondary and not args.outdir_secondary): # pragma: no cover logging.error( 'For writing secondary json test results, a secondary outdir ' 'patch must be specified.') return INFRA_FAILURE_RETCODE workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) if args.buildbot: build_config = 'Release' else: build_config = '%s.release' % args.arch if args.binary_override_path == None: args.shell_dir = os.path.join(workspace, args.outdir, build_config) default_binary_name = 'd8' else: if not os.path.isfile(args.binary_override_path): logging.error('binary-override-path must be a file name') return INFRA_FAILURE_RETCODE if args.outdir_secondary: logging.error( 'specify either binary-override-path or outdir-secondary') return INFRA_FAILURE_RETCODE args.shell_dir = os.path.abspath( os.path.dirname(args.binary_override_path)) default_binary_name = os.path.basename(args.binary_override_path) if args.outdir_secondary: args.shell_dir_secondary = os.path.join(workspace, args.outdir_secondary, build_config) else: args.shell_dir_secondary = None if args.json_test_results: args.json_test_results = os.path.abspath(args.json_test_results) if args.json_test_results_secondary: args.json_test_results_secondary = os.path.abspath( args.json_test_results_secondary) # Ensure all arguments have absolute path before we start changing current # directory. args.suite = map(os.path.abspath, args.suite) prev_aslr = None prev_cpu_gov = None platform = Platform.GetPlatform(args) result_tracker = ResultTracker() result_tracker_secondary = ResultTracker() have_failed_tests = False with CustomMachineConfiguration(governor=args.cpu_governor, disable_aslr=args.noaslr) as conf: for path in args.suite: if not os.path.exists(path): # pragma: no cover result_tracker.AddError( 'Configuration file %s does not exist.' % path) continue with open(path) as f: suite = json.loads(f.read()) # If no name is given, default to the file name without .json. suite.setdefault('name', os.path.splitext(os.path.basename(path))[0]) # Setup things common to one test suite. platform.PreExecution() # Build the graph/trace tree structure. default_parent = DefaultSentinel(default_binary_name) root = BuildGraphConfigs(suite, args.arch, default_parent) # Callback to be called on each node on traversal. def NodeCB(node): platform.PreTests(node, path) # Traverse graph/trace tree and iterate over all runnables. start = time.time() try: for runnable in FlattenRunnables(root, NodeCB): runnable_name = '/'.join(runnable.graphs) if (not runnable_name.startswith(args.filter) and runnable_name + '/' != args.filter): continue logging.info('>>> Running suite: %s', runnable_name) def RunGenerator(runnable): if args.confidence_level: counter = 0 while not result_tracker.HasEnoughRuns( runnable, args.confidence_level): yield counter counter += 1 else: for i in range( 0, max(1, args.run_count or runnable.run_count)): yield i for i in RunGenerator(runnable): attempts_left = runnable.retry_count + 1 while attempts_left: total_duration = time.time() - start if total_duration > args.max_total_duration: logging.info( '>>> Stopping now since running for too long (%ds > %ds)', total_duration, args.max_total_duration) raise MaxTotalDurationReachedError() output, output_secondary = platform.Run( runnable, i, secondary=args.shell_dir_secondary) result_tracker.AddRunnableDuration( runnable, output.duration) result_tracker_secondary.AddRunnableDuration( runnable, output_secondary.duration) if output.IsSuccess( ) and output_secondary.IsSuccess(): runnable.ProcessOutput(output, result_tracker, i) if output_secondary is not NULL_OUTPUT: runnable.ProcessOutput( output_secondary, result_tracker_secondary, i) break attempts_left -= 1 if not attempts_left: logging.info( '>>> Suite %s failed after %d retries', runnable_name, runnable.retry_count + 1) have_failed_tests = True else: logging.info('>>> Retrying suite: %s', runnable_name) except MaxTotalDurationReachedError: have_failed_tests = True platform.PostExecution() if args.json_test_results: result_tracker.WriteToFile(args.json_test_results) else: # pragma: no cover print('Primary results:', result_tracker) if args.shell_dir_secondary: if args.json_test_results_secondary: result_tracker_secondary.WriteToFile( args.json_test_results_secondary) else: # pragma: no cover print('Secondary results:', result_tracker_secondary) if (result_tracker.errors or result_tracker_secondary.errors or have_failed_tests): return 1 return 0
def Main(argv): parser = argparse.ArgumentParser() parser.add_argument('--arch', help='The architecture to run tests for. Pass "auto" ' 'to auto-detect.', default='x64', choices=SUPPORTED_ARCHS + ['auto']) parser.add_argument('--buildbot', help='Adapt to path structure used on buildbots and adds ' 'timestamps/level to all logged status messages', default=False, action='store_true') parser.add_argument('-d', '--device', help='The device ID to run Android tests on. If not ' 'given it will be autodetected.') parser.add_argument('--extra-flags', help='Additional flags to pass to the test executable', default='') parser.add_argument('--json-test-results', help='Path to a file for storing json results.') parser.add_argument('--json-test-results-secondary', help='Path to a file for storing json results from run ' 'without patch or for reference build run.') parser.add_argument('--outdir', help='Base directory with compile output', default='out') parser.add_argument('--outdir-secondary', help='Base directory with compile output without patch ' 'or for reference build') parser.add_argument('--binary-override-path', help='JavaScript engine binary. By default, d8 under ' 'architecture-specific build dir. ' 'Not supported in conjunction with outdir-secondary.') parser.add_argument('--prioritize', help='Raise the priority to nice -20 for the ' 'benchmarking process.Requires Linux, schedtool, and ' 'sudo privileges.', default=False, action='store_true') parser.add_argument('--affinitize', help='Run benchmarking process on the specified core. ' 'For example: --affinitize=0 will run the benchmark ' 'process on core 0. --affinitize=3 will run the ' 'benchmark process on core 3. Requires Linux, schedtool, ' 'and sudo privileges.', default=None) parser.add_argument('--noaslr', help='Disable ASLR for the duration of the benchmarked ' 'process. Requires Linux and sudo privileges.', default=False, action='store_true') parser.add_argument('--cpu-governor', help='Set cpu governor to specified policy for the ' 'duration of the benchmarked process. Typical options: ' '"powersave" for more stable results, or "performance" ' 'for shorter completion time of suite, with potentially ' 'more noise in results.') parser.add_argument('--filter', help='Only run the benchmarks beginning with this ' 'string. For example: ' '--filter=JSTests/TypedArrays/ will run only TypedArray ' 'benchmarks from the JSTests suite.', default='') parser.add_argument('--dump-logcats-to', help='Writes logcat output from each test into specified ' 'directory. Only supported for android targets.') parser.add_argument("--run-count", type=int, default=0, help="Override the run count specified by the test " "suite. The default 0 uses the suite's config.") parser.add_argument('suite', nargs='+', help='Path to the suite config file.') try: args = parser.parse_args(argv) except SystemExit: return INFRA_FAILURE_RETCODE logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s') if args.arch == 'auto': # pragma: no cover args.arch = utils.DefaultArch() if args.arch not in SUPPORTED_ARCHS: logging.error( 'Auto-detected architecture "%s" is not supported.', args.arch) return INFRA_FAILURE_RETCODE if (args.json_test_results_secondary and not args.outdir_secondary): # pragma: no cover logging.error('For writing secondary json test results, a secondary outdir ' 'patch must be specified.') return INFRA_FAILURE_RETCODE workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) if args.buildbot: build_config = 'Release' else: build_config = '%s.release' % args.arch if args.binary_override_path == None: args.shell_dir = os.path.join(workspace, args.outdir, build_config) default_binary_name = 'd8' else: if not os.path.isfile(args.binary_override_path): logging.error('binary-override-path must be a file name') return INFRA_FAILURE_RETCODE if args.outdir_secondary: logging.error('specify either binary-override-path or outdir-secondary') return INFRA_FAILURE_RETCODE args.shell_dir = os.path.abspath( os.path.dirname(args.binary_override_path)) default_binary_name = os.path.basename(args.binary_override_path) if args.outdir_secondary: args.shell_dir_secondary = os.path.join( workspace, args.outdir_secondary, build_config) else: args.shell_dir_secondary = None if args.json_test_results: args.json_test_results = os.path.abspath(args.json_test_results) if args.json_test_results_secondary: args.json_test_results_secondary = os.path.abspath( args.json_test_results_secondary) # Ensure all arguments have absolute path before we start changing current # directory. args.suite = map(os.path.abspath, args.suite) prev_aslr = None prev_cpu_gov = None platform = Platform.GetPlatform(args) results = Results() results_secondary = Results() # We use list here to allow modification in nested function below. have_failed_tests = [False] with CustomMachineConfiguration(governor = args.cpu_governor, disable_aslr = args.noaslr) as conf: for path in args.suite: if not os.path.exists(path): # pragma: no cover results.errors.append('Configuration file %s does not exist.' % path) continue with open(path) as f: suite = json.loads(f.read()) # If no name is given, default to the file name without .json. suite.setdefault('name', os.path.splitext(os.path.basename(path))[0]) # Setup things common to one test suite. platform.PreExecution() # Build the graph/trace tree structure. default_parent = DefaultSentinel(default_binary_name) root = BuildGraphConfigs(suite, args.arch, default_parent) # Callback to be called on each node on traversal. def NodeCB(node): platform.PreTests(node, path) # Traverse graph/trace tree and iterate over all runnables. for runnable in FlattenRunnables(root, NodeCB): runnable_name = '/'.join(runnable.graphs) if (not runnable_name.startswith(args.filter) and runnable_name + '/' != args.filter): continue logging.info('>>> Running suite: %s', runnable_name) durations = [] durations_secondary = [] def Runner(): """Output generator that reruns several times.""" for i in range(0, max(1, args.run_count or runnable.run_count)): attempts_left = runnable.retry_count + 1 while attempts_left: output, output_secondary = platform.Run(runnable, i) if output.IsSuccess() and output_secondary.IsSuccess(): durations.append(output.duration) if output_secondary is not NULL_OUTPUT: durations_secondary.append(output_secondary.duration) yield output, output_secondary break attempts_left -= 1 if not attempts_left: # ignore failures until last attempt have_failed_tests[0] = True else: logging.info('>>> Retrying suite: %s', runnable_name) # Let runnable iterate over all runs and handle output. result, result_secondary = runnable.Run( Runner, trybot=args.shell_dir_secondary) results += result results_secondary += result_secondary if runnable.has_timeouts: results.timeouts.append(runnable_name) if runnable.has_near_timeouts: results.near_timeouts.append(runnable_name) results.runnable_durations.append({ 'graphs': runnable.graphs, 'durations': durations, 'timeout': runnable.timeout, }) if durations_secondary: results_secondary.runnable_durations.append({ 'graphs': runnable.graphs, 'durations': durations_secondary, 'timeout': runnable.timeout, }) platform.PostExecution() if args.json_test_results: results.WriteToFile(args.json_test_results) else: # pragma: no cover print(results) if args.json_test_results_secondary: results_secondary.WriteToFile(args.json_test_results_secondary) else: # pragma: no cover print(results_secondary) if results.errors or have_failed_tests[0]: return 1 return 0