def make_fake_experiment(self):
test_flag.SetTestMode(True)
experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1))
experiment = ExperimentFactory().GetExperiment(experiment_file,
working_directory='',
log_dir='')
return experiment
def test_telemetry_run(self, mock_runcmd, mock_exists, mock_isdir):
def FakeLogMsg(fd, termfd, msg, flush=True):
if fd or termfd or msg or flush:
pass
save_log_msg = self.real_logger.LogMsg
self.real_logger.LogMsg = FakeLogMsg
mock_runcmd.return_value = 0
self.mock_cmd_exec.RunCommandWOutput = mock_runcmd
self.runner.logger = self.real_logger
profiler_args = (
'--profiler=custom_perf --profiler_args=\'perf_options'
'="record -a -e cycles,instructions"\'')
raises_exception = False
mock_isdir.return_value = False
try:
self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
self.telemetry_bench, '')
except SystemExit:
raises_exception = True
self.assertTrue(raises_exception)
raises_exception = False
mock_isdir.return_value = True
mock_exists.return_value = False
try:
self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
self.telemetry_bench, '')
except SystemExit:
raises_exception = True
self.assertTrue(raises_exception)
raises_exception = False
mock_isdir.return_value = True
mock_exists.return_value = True
try:
self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
self.telemetry_bench, profiler_args)
except SystemExit:
raises_exception = True
self.assertTrue(raises_exception)
test_flag.SetTestMode(True)
res = self.runner.Telemetry_Run('lumpy1.cros', self.mock_label,
self.telemetry_bench, '')
self.assertEqual(res, 0)
self.assertEqual(mock_runcmd.call_count, 1)
self.assertEqual(
mock_runcmd.call_args_list[0][0],
(('cd src/tools/perf && ./run_measurement '
'--browser=cros-chrome --output-format=csv '
'--remote=lumpy1.cros --identity /tmp/chromeos/src/scripts'
'/mod_for_test_scripts/ssh_keys/testing_rsa octane '), ))
self.real_logger.LogMsg = save_log_msg
def RunCrosperf(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--noschedv2',
dest='noschedv2',
default=False,
action='store_true',
help=('Do not use new scheduler. '
'Use original scheduler instead.'))
parser.add_argument(
'-l',
'--log_dir',
dest='log_dir',
default='',
help='The log_dir, default is under <crosperf_logs>/logs')
SetupParserOptions(parser)
options, args = parser.parse_known_args(argv)
# Convert the relevant options that are passed in into a settings
# object which will override settings in the experiment file.
option_settings = ConvertOptionsToSettings(options)
log_dir = os.path.abspath(os.path.expanduser(options.log_dir))
logger.GetLogger(log_dir)
if len(args) == 2:
experiment_filename = args[1]
else:
parser.error('Invalid number arguments.')
working_directory = os.getcwd()
if options.dry_run:
test_flag.SetTestMode(True)
experiment_file = ExperimentFile(open(experiment_filename, 'rb'),
option_settings)
if not experiment_file.GetGlobalSettings().GetField('name'):
experiment_name = os.path.basename(experiment_filename)
experiment_file.GetGlobalSettings().SetField('name', experiment_name)
experiment = ExperimentFactory().GetExperiment(experiment_file,
working_directory, log_dir)
json_report = experiment_file.GetGlobalSettings().GetField('json_report')
signal.signal(signal.SIGTERM, CallExitHandler)
atexit.register(Cleanup, experiment)
if options.dry_run:
runner = MockExperimentRunner(experiment, json_report)
else:
runner = ExperimentRunner(experiment,
json_report,
using_schedv2=(not options.noschedv2))
runner.Run()
def test_test_flag(self):
# Verify that test_flag.is_test exists, that it is a list,
# and that it contains 1 element.
self.assertTrue(type(test_flag.is_test) is list)
self.assertEqual(len(test_flag.is_test), 1)
# Verify that the getting the flag works and that the flag
# contains False, its starting value.
save_flag = test_flag.GetTestMode()
self.assertFalse(save_flag)
# Verify that setting the flat to True, then getting it, works.
test_flag.SetTestMode(True)
self.assertTrue(test_flag.GetTestMode())
# Verify that setting the flag to False, then getting it, works.
test_flag.SetTestMode(save_flag)
self.assertFalse(test_flag.GetTestMode())
# Verify that test_flag.is_test still exists, that it still is a
# list, and that it still contains 1 element.
self.assertTrue(type(test_flag.is_test) is list)
self.assertEqual(len(test_flag.is_test), 1)
self.exp = self._make_fake_experiment(
EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30))
my_schedv2 = Schedv2(self.exp)
self.assertEquals(len(my_schedv2.get_cached_run_list()), 30)
# The non-cache-hit brs are put into Schedv2._label_brl_map.
self.assertEquals(
reduce(lambda a, x: a + len(x[1]),
my_schedv2.get_label_map().iteritems(), 0), 30)
def test_nocachehit(self):
"""Test no cache-hit."""
def MockReadCache(br):
br.cache_hit = False
with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache',
new=MockReadCache):
# We have 2 * 30 brs, none of which are put into _cached_br_list.
self.exp = self._make_fake_experiment(
EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30))
my_schedv2 = Schedv2(self.exp)
self.assertEquals(len(my_schedv2.get_cached_run_list()), 0)
# The non-cache-hit brs are put into Schedv2._label_brl_map.
self.assertEquals(
reduce(lambda a, x: a + len(x[1]),
my_schedv2.get_label_map().iteritems(), 0), 60)
if __name__ == '__main__':
test_flag.SetTestMode(True)
unittest.main()
def test_get_experiment(self, mock_socket):
test_flag.SetTestMode(False)
self.append_benchmark_call_args = []
def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch,
perf_args, suite, show_all):
'Helper function for test_get_experiment'
arg_list = [
bench_list, set_list, args, iters, rm_ch, perf_args, suite,
show_all
]
self.append_benchmark_call_args.append(arg_list)
def FakeGetDefaultRemotes(board):
if not board:
return []
return [
'fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'
]
def FakeGetXbuddyPath(build, autotest_dir, board, chroot, log_level):
autotest_path = autotest_dir
if not autotest_path:
autotest_path = 'fake_autotest_path'
if not build or not board or not chroot or not log_level:
return '', autotest_path
return 'fake_image_path', autotest_path
ef = ExperimentFactory()
ef.AppendBenchmarkSet = FakeAppendBenchmarkSet
ef.GetDefaultRemotes = FakeGetDefaultRemotes
label_settings = settings_factory.LabelSettings('image_label')
benchmark_settings = settings_factory.BenchmarkSettings('bench_test')
global_settings = settings_factory.GlobalSettings('test_name')
label_settings.GetXbuddyPath = FakeGetXbuddyPath
mock_experiment_file = ExperimentFile(StringIO.StringIO(''))
mock_experiment_file.all_settings = []
test_flag.SetTestMode(True)
# Basic test.
global_settings.SetField('name', 'unittest_test')
global_settings.SetField('board', 'lumpy')
global_settings.SetField('remote', '123.45.67.89 123.45.76.80')
benchmark_settings.SetField('test_name', 'kraken')
benchmark_settings.SetField('suite', 'telemetry_Crosperf')
benchmark_settings.SetField('iterations', 1)
label_settings.SetField(
'chromeos_image',
'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin')
label_settings.SetField('chrome_src',
'/usr/local/google/home/chrome-top')
label_settings.SetField('autotest_path', '/tmp/autotest')
mock_experiment_file.global_settings = global_settings
mock_experiment_file.all_settings.append(label_settings)
mock_experiment_file.all_settings.append(benchmark_settings)
mock_experiment_file.all_settings.append(global_settings)
mock_socket.return_value = ''
# First test. General test.
exp = ef.GetExperiment(mock_experiment_file, '', '')
self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80'])
self.assertEqual(exp.cache_conditions, [0, 2, 1])
self.assertEqual(exp.log_level, 'average')
self.assertEqual(len(exp.benchmarks), 1)
self.assertEqual(exp.benchmarks[0].name, 'kraken')
self.assertEqual(exp.benchmarks[0].test_name, 'kraken')
self.assertEqual(exp.benchmarks[0].iterations, 1)
self.assertEqual(exp.benchmarks[0].suite, 'telemetry_Crosperf')
self.assertFalse(exp.benchmarks[0].show_all_results)
self.assertEqual(len(exp.labels), 1)
self.assertEqual(
exp.labels[0].chromeos_image,
'chromeos/src/build/images/lumpy/latest/'
'chromiumos_test_image.bin')
self.assertEqual(exp.labels[0].autotest_path, '/tmp/autotest')
self.assertEqual(exp.labels[0].board, 'lumpy')
# Second test: Remotes listed in labels.
test_flag.SetTestMode(True)
label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros')
exp = ef.GetExperiment(mock_experiment_file, '', '')
self.assertEqual(exp.remote, [
'chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80'
])
# Third test: Automatic fixing of bad logging_level param:
global_settings.SetField('logging_level', 'really loud!')
exp = ef.GetExperiment(mock_experiment_file, '', '')
self.assertEqual(exp.log_level, 'verbose')
# Fourth test: Setting cache conditions; only 1 remote with "same_machine"
global_settings.SetField('rerun_if_failed', 'true')
global_settings.SetField('rerun', 'true')
global_settings.SetField('same_machine', 'true')
global_settings.SetField('same_specs', 'true')
self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file,
'', '')
label_settings.SetField('remote', '')
global_settings.SetField('remote', '123.45.67.89')
exp = ef.GetExperiment(mock_experiment_file, '', '')
self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1])
# Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all
# remotes (Call GetDefaultRemotes).
mock_socket.return_value = 'test.corp.google.com'
global_settings.SetField('remote', '')
global_settings.SetField('same_machine', 'false')
label_settings_2 = settings_factory.LabelSettings(
'official_image_label')
label_settings_2.SetField('chromeos_root', 'chromeos')
label_settings_2.SetField('build', 'official-dev')
label_settings_2.SetField('autotest_path', '')
label_settings_2.GetXbuddyPath = FakeGetXbuddyPath
mock_experiment_file.all_settings.append(label_settings_2)
exp = ef.GetExperiment(mock_experiment_file, '', '')
self.assertEqual(len(exp.labels), 2)
self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path')
self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path')
self.assertEqual(
exp.remote,
['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'])
def test_download_image(self, mock_path_exists, mock_mkdirs):
# Set mock and test values.
mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter)
test_chroot = '/usr/local/home/chromeos'
test_build_id = 'lumpy-release/R36-5814.0.0'
image_path = ('gs://chromeos-image-archive/%s/chromiumos_test_image.tar.xz'
% test_build_id)
downloader = download_images.ImageDownloader(
logger_to_use=MOCK_LOGGER, cmd_exec=mock_cmd_exec)
# Set os.path.exists to always return False and run downloader
mock_path_exists.return_value = False
test_flag.SetTestMode(True)
self.assertRaises(download_images.MissingImage, downloader.DownloadImage,
test_chroot, test_build_id, image_path)
# Verify os.path.exists was called twice, with proper arguments.
self.assertEqual(mock_path_exists.call_count, 2)
mock_path_exists.assert_called_with(
'/usr/local/home/chromeos/chroot/tmp/lumpy-release/'
'R36-5814.0.0/chromiumos_test_image.bin')
mock_path_exists.assert_any_call(
'/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0')
# Verify we called os.mkdirs
self.assertEqual(mock_mkdirs.call_count, 1)
mock_mkdirs.assert_called_with(
'/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0')
# Verify we called RunCommand once, with proper arguments.
self.assertEqual(mock_cmd_exec.RunCommand.call_count, 1)
expected_args = (
'/usr/local/home/chromeos/chromium/tools/depot_tools/gsutil.py '
'cp gs://chromeos-image-archive/lumpy-release/R36-5814.0.0/'
'chromiumos_test_image.tar.xz '
'/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0')
mock_cmd_exec.RunCommand.assert_called_with(expected_args)
# Reset the velues in the mocks; set os.path.exists to always return True.
mock_path_exists.reset_mock()
mock_cmd_exec.reset_mock()
mock_path_exists.return_value = True
# Run downloader
downloader.DownloadImage(test_chroot, test_build_id, image_path)
# Verify os.path.exists was called twice, with proper arguments.
self.assertEqual(mock_path_exists.call_count, 2)
mock_path_exists.assert_called_with(
'/usr/local/home/chromeos/chroot/tmp/lumpy-release/'
'R36-5814.0.0/chromiumos_test_image.bin')
mock_path_exists.assert_any_call(
'/usr/local/home/chromeos/chroot/tmp/lumpy-release/R36-5814.0.0')
# Verify we made no RunCommand or ChrootRunCommand calls (since
# os.path.exists returned True, there was no work do be done).
self.assertEqual(mock_cmd_exec.RunCommand.call_count, 0)
self.assertEqual(mock_cmd_exec.ChrootRunCommand.call_count, 0)
def test_image_machine(self, mock_checksummer, mock_run_croscmd,
mock_run_cmd, mock_sleep):
def FakeMD5Checksum(_input_str):
return 'machine_fake_md5_checksum'
self.fake_logger_count = 0
self.fake_logger_msgs = []
def FakeLogOutput(msg):
self.fake_logger_count += 1
self.fake_logger_msgs.append(msg)
def ResetValues():
self.fake_logger_count = 0
self.fake_logger_msgs = []
mock_run_cmd.reset_mock()
mock_run_croscmd.reset_mock()
mock_checksummer.reset_mock()
mock_sleep.reset_mock()
machine.checksum = 'fake_md5_checksum'
self.mm.checksum = None
self.mm.num_reimages = 0
self.mock_cmd_exec.CrosRunCommand = mock_run_croscmd
self.mock_cmd_exec.RunCommand = mock_run_cmd
self.mm.logger.LogOutput = FakeLogOutput
machine = self.mock_lumpy1
machine._GetMD5Checksum = FakeMD5Checksum
machine.checksum = 'fake_md5_checksum'
mock_checksummer.return_value = 'fake_md5_checksum'
self.mock_cmd_exec.log_level = 'verbose'
test_flag.SetTestMode(True)
# Test 1: label.image_type == "local"
LABEL_LUMPY.image_type = 'local'
self.mm.ImageMachine(machine, LABEL_LUMPY)
self.assertEqual(mock_run_cmd.call_count, 0)
self.assertEqual(mock_run_croscmd.call_count, 0)
#Test 2: label.image_type == "trybot"
ResetValues()
LABEL_LUMPY.image_type = 'trybot'
mock_run_cmd.return_value = 0
self.mm.ImageMachine(machine, LABEL_LUMPY)
self.assertEqual(mock_run_croscmd.call_count, 0)
self.assertEqual(mock_checksummer.call_count, 0)
# Test 3: label.image_type is neither local nor trybot; retval from
# RunCommand is 1, i.e. image_chromeos fails...
ResetValues()
LABEL_LUMPY.image_type = 'other'
mock_run_cmd.return_value = 1
try:
self.mm.ImageMachine(machine, LABEL_LUMPY)
except RuntimeError:
self.assertEqual(mock_checksummer.call_count, 0)
self.assertEqual(mock_run_cmd.call_count, 2)
self.assertEqual(mock_run_croscmd.call_count, 1)
self.assertEqual(mock_sleep.call_count, 1)
image_call_args_str = mock_run_cmd.call_args[0][0]
image_call_args = image_call_args_str.split(' ')
self.assertEqual(image_call_args[0], 'python')
self.assertEqual(image_call_args[1].split('/')[-1],
'image_chromeos.pyc')
image_call_args = image_call_args[2:]
self.assertEqual(image_call_args, [
'--chromeos_root=/tmp/chromeos-root',
'--image=lumpy_chromeos_image', '--image_args=',
'--remote=lumpy1', '--logging_level=average', '--board=lumpy'
])
self.assertEqual(mock_run_croscmd.call_args[0][0],
'reboot && exit')
# Test 4: Everything works properly. Trybot image type.
ResetValues()
LABEL_LUMPY.image_type = 'trybot'
mock_run_cmd.return_value = 0
self.mm.ImageMachine(machine, LABEL_LUMPY)
self.assertEqual(mock_checksummer.call_count, 0)
self.assertEqual(mock_run_croscmd.call_count, 0)
self.assertEqual(mock_sleep.call_count, 0)