def make_fake_experiment(self): test_flag.SetTestMode(True) experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) experiment = ExperimentFactory().GetExperiment(experiment_file, working_directory='', log_dir='') return experiment
def MakeMockExperiment(compiler='gcc'): """Mocks an experiment using the given compiler.""" mock_experiment_file = StringIO(""" board: x86-alex remote: 127.0.0.1 perf_args: record -a -e cycles benchmark: PageCycler { iterations: 3 } image1 { chromeos_image: %s } image2 { remote: 127.0.0.2 chromeos_image: %s } """ % (FakePath('cros_image1.bin'), FakePath('cros_image2.bin'))) efile = ExperimentFile(mock_experiment_file) experiment = ExperimentFactory().GetExperiment(efile, FakePath('working_directory'), FakePath('log_dir')) for label in experiment.labels: label.compiler = compiler return experiment
def _make_fake_experiment(self, expstr): """Create fake experiment from string. Note - we mock out BenchmarkRun in this step. """ experiment_file = ExperimentFile(StringIO.StringIO(expstr)) experiment = ExperimentFactory().GetExperiment(experiment_file, working_directory='', log_dir='') return experiment
def RunCrosperf(argv): parser = argparse.ArgumentParser() parser.add_argument('--noschedv2', dest='noschedv2', default=False, action='store_true', help=('Do not use new scheduler. ' 'Use original scheduler instead.')) parser.add_argument( '-l', '--log_dir', dest='log_dir', default='', help='The log_dir, default is under <crosperf_logs>/logs') SetupParserOptions(parser) options, args = parser.parse_known_args(argv) # Convert the relevant options that are passed in into a settings # object which will override settings in the experiment file. option_settings = ConvertOptionsToSettings(options) log_dir = os.path.abspath(os.path.expanduser(options.log_dir)) logger.GetLogger(log_dir) if len(args) == 2: experiment_filename = args[1] else: parser.error('Invalid number arguments.') working_directory = os.getcwd() if options.dry_run: test_flag.SetTestMode(True) experiment_file = ExperimentFile(open(experiment_filename, 'rb'), option_settings) if not experiment_file.GetGlobalSettings().GetField('name'): experiment_name = os.path.basename(experiment_filename) experiment_file.GetGlobalSettings().SetField('name', experiment_name) experiment = ExperimentFactory().GetExperiment(experiment_file, working_directory, log_dir) json_report = experiment_file.GetGlobalSettings().GetField('json_report') signal.signal(signal.SIGTERM, CallExitHandler) atexit.register(Cleanup, experiment) if options.dry_run: runner = MockExperimentRunner(experiment, json_report) else: runner = ExperimentRunner(experiment, json_report, using_schedv2=(not options.noschedv2)) runner.Run()
def testOverrideSetting(self): input_file = StringIO.StringIO(EXPERIMENT_FILE_2) experiment_file = ExperimentFile(input_file) global_settings = experiment_file.GetGlobalSettings() self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3']) benchmark_settings = experiment_file.GetSettings('benchmark') self.assertEqual(len(benchmark_settings), 2) self.assertEqual(benchmark_settings[0].name, 'PageCycler') self.assertEqual(benchmark_settings[0].GetField('iterations'), 3) self.assertEqual(benchmark_settings[1].name, 'AndroidBench') self.assertEqual(benchmark_settings[1].GetField('iterations'), 2)
def testLoadExperimentFile1(self): experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) exp = ExperimentFactory().GetExperiment( experiment_file, working_directory='', log_dir='') self.assertEqual(exp.remote, ['chromeos-alex3']) self.assertEqual(len(exp.benchmarks), 1) self.assertEqual(exp.benchmarks[0].name, 'PageCycler') self.assertEqual(exp.benchmarks[0].test_name, 'PageCycler') self.assertEqual(exp.benchmarks[0].iterations, 3) self.assertEqual(len(exp.labels), 2) self.assertEqual(exp.labels[0].chromeos_image, '/usr/local/google/cros_image1.bin') self.assertEqual(exp.labels[0].board, 'x86-alex')
def testLoadExperimentFile1(self): input_file = StringIO.StringIO(EXPERIMENT_FILE_1) experiment_file = ExperimentFile(input_file) global_settings = experiment_file.GetGlobalSettings() self.assertEqual(global_settings.GetField('remote'), ['chromeos-alex3']) self.assertEqual(global_settings.GetField('perf_args'), 'record -a -e cycles') benchmark_settings = experiment_file.GetSettings('benchmark') self.assertEqual(len(benchmark_settings), 1) self.assertEqual(benchmark_settings[0].name, 'PageCycler') self.assertEqual(benchmark_settings[0].GetField('iterations'), 3) label_settings = experiment_file.GetSettings('label') self.assertEqual(len(label_settings), 2) self.assertEqual(label_settings[0].name, 'image1') self.assertEqual(label_settings[0].GetField('chromeos_image'), '/usr/local/google/cros_image1.bin') self.assertEqual(label_settings[1].GetField('remote'), ['chromeos-lumpy1']) self.assertEqual(label_settings[0].GetField('remote'), ['chromeos-alex3'])
def test_get_experiment(self, mock_socket): test_flag.SetTestMode(False) self.append_benchmark_call_args = [] def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch, perf_args, suite, show_all): 'Helper function for test_get_experiment' arg_list = [ bench_list, set_list, args, iters, rm_ch, perf_args, suite, show_all ] self.append_benchmark_call_args.append(arg_list) def FakeGetDefaultRemotes(board): if not board: return [] return [ 'fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros' ] def FakeGetXbuddyPath(build, autotest_dir, board, chroot, log_level): autotest_path = autotest_dir if not autotest_path: autotest_path = 'fake_autotest_path' if not build or not board or not chroot or not log_level: return '', autotest_path return 'fake_image_path', autotest_path ef = ExperimentFactory() ef.AppendBenchmarkSet = FakeAppendBenchmarkSet ef.GetDefaultRemotes = FakeGetDefaultRemotes label_settings = settings_factory.LabelSettings('image_label') benchmark_settings = settings_factory.BenchmarkSettings('bench_test') global_settings = settings_factory.GlobalSettings('test_name') label_settings.GetXbuddyPath = FakeGetXbuddyPath mock_experiment_file = ExperimentFile(StringIO.StringIO('')) mock_experiment_file.all_settings = [] test_flag.SetTestMode(True) # Basic test. global_settings.SetField('name', 'unittest_test') global_settings.SetField('board', 'lumpy') global_settings.SetField('remote', '123.45.67.89 123.45.76.80') benchmark_settings.SetField('test_name', 'kraken') benchmark_settings.SetField('suite', 'telemetry_Crosperf') benchmark_settings.SetField('iterations', 1) label_settings.SetField( 'chromeos_image', 'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin') label_settings.SetField('chrome_src', '/usr/local/google/home/chrome-top') label_settings.SetField('autotest_path', '/tmp/autotest') mock_experiment_file.global_settings = global_settings mock_experiment_file.all_settings.append(label_settings) mock_experiment_file.all_settings.append(benchmark_settings) mock_experiment_file.all_settings.append(global_settings) mock_socket.return_value = '' # First test. General test. exp = ef.GetExperiment(mock_experiment_file, '', '') self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80']) self.assertEqual(exp.cache_conditions, [0, 2, 1]) self.assertEqual(exp.log_level, 'average') self.assertEqual(len(exp.benchmarks), 1) self.assertEqual(exp.benchmarks[0].name, 'kraken') self.assertEqual(exp.benchmarks[0].test_name, 'kraken') self.assertEqual(exp.benchmarks[0].iterations, 1) self.assertEqual(exp.benchmarks[0].suite, 'telemetry_Crosperf') self.assertFalse(exp.benchmarks[0].show_all_results) self.assertEqual(len(exp.labels), 1) self.assertEqual( exp.labels[0].chromeos_image, 'chromeos/src/build/images/lumpy/latest/' 'chromiumos_test_image.bin') self.assertEqual(exp.labels[0].autotest_path, '/tmp/autotest') self.assertEqual(exp.labels[0].board, 'lumpy') # Second test: Remotes listed in labels. test_flag.SetTestMode(True) label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros') exp = ef.GetExperiment(mock_experiment_file, '', '') self.assertEqual(exp.remote, [ 'chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80' ]) # Third test: Automatic fixing of bad logging_level param: global_settings.SetField('logging_level', 'really loud!') exp = ef.GetExperiment(mock_experiment_file, '', '') self.assertEqual(exp.log_level, 'verbose') # Fourth test: Setting cache conditions; only 1 remote with "same_machine" global_settings.SetField('rerun_if_failed', 'true') global_settings.SetField('rerun', 'true') global_settings.SetField('same_machine', 'true') global_settings.SetField('same_specs', 'true') self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '') label_settings.SetField('remote', '') global_settings.SetField('remote', '123.45.67.89') exp = ef.GetExperiment(mock_experiment_file, '', '') self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1]) # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all # remotes (Call GetDefaultRemotes). mock_socket.return_value = 'test.corp.google.com' global_settings.SetField('remote', '') global_settings.SetField('same_machine', 'false') label_settings_2 = settings_factory.LabelSettings( 'official_image_label') label_settings_2.SetField('chromeos_root', 'chromeos') label_settings_2.SetField('build', 'official-dev') label_settings_2.SetField('autotest_path', '') label_settings_2.GetXbuddyPath = FakeGetXbuddyPath mock_experiment_file.all_settings.append(label_settings_2) exp = ef.GetExperiment(mock_experiment_file, '', '') self.assertEqual(len(exp.labels), 2) self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path') self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path') self.assertEqual( exp.remote, ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'])
def testCanonicalize(self): input_file = StringIO.StringIO(EXPERIMENT_FILE_1) experiment_file = ExperimentFile(input_file) res = experiment_file.Canonicalize() self.assertEqual(res, OUTPUT_FILE)