def test_terminate_fail(self):
br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
self.test_label, 1,
self.test_cache_conditions,
self.mock_machine_manager,
self.mock_logger, 'average', '')
def GetLastEventFailed():
'Helper function for test_terminate_fail'
return benchmark_run.STATUS_FAILED
def RecordStub(status):
'Helper function for test_terminate_fail'
self.status = status
self.status = benchmark_run.STATUS_SUCCEEDED
self.assertFalse(br.terminated)
self.assertFalse(br.suite_runner.CommandTerminator().IsTerminated())
br.timeline.GetLastEvent = GetLastEventFailed
br.timeline.Record = RecordStub
br.Terminate()
self.assertTrue(br.terminated)
self.assertTrue(br.suite_runner.CommandTerminator().IsTerminated())
self.assertEqual(self.status, benchmark_run.STATUS_SUCCEEDED)
def test_set_cache_conditions(self):
br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
self.test_label, 1,
self.test_cache_conditions,
self.mock_machine_manager,
self.mock_logger, 'average', '')
phony_cache_conditions = [123, 456, True, False]
self.assertEqual(br.cache_conditions, self.test_cache_conditions)
br.SetCacheConditions(phony_cache_conditions)
self.assertEqual(br.cache_conditions, phony_cache_conditions)
br.SetCacheConditions(self.test_cache_conditions)
self.assertEqual(br.cache_conditions, self.test_cache_conditions)
def test_acquire_machine(self):
br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
self.test_label, 1,
self.test_cache_conditions,
self.mock_machine_manager,
self.mock_logger, 'average', '')
br.terminated = True
self.assertRaises(Exception, br.AcquireMachine)
br.terminated = False
mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
'chromeos', 'average')
self.mock_machine_manager.AcquireMachine.return_value = mock_machine
machine = br.AcquireMachine()
self.assertEqual(machine.name, 'chromeos1-row3-rack5-host7.cros')
def _GenerateBenchmarkRuns(self):
"""Generate benchmark runs from labels and benchmark defintions."""
benchmark_runs = []
for label in self.labels:
for benchmark in self.benchmarks:
for iteration in xrange(1, benchmark.iterations + 1):
benchmark_run_name = '%s: %s (%s)' % (label.name, benchmark.name,
iteration)
full_name = '%s_%s_%s' % (label.name, benchmark.name, iteration)
logger_to_use = logger.Logger(self.log_dir, 'run.%s' % (full_name),
True)
benchmark_runs.append(
benchmark_run.BenchmarkRun(benchmark_run_name, benchmark, label,
iteration, self.cache_conditions,
self.machine_manager, logger_to_use,
self.log_level, self.share_cache))
return benchmark_runs
def test_get_extra_autotest_args(self):
br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
self.test_label, 1,
self.test_cache_conditions,
self.mock_machine_manager,
self.mock_logger, 'average', '')
def MockLogError(err_msg):
'Helper function for test_get_extra_autotest_args'
self.err_msg = err_msg
self.mock_logger.LogError = MockLogError
result = br.GetExtraAutotestArgs()
self.assertEqual(result, '')
self.test_benchmark.perf_args = 'record -e cycles'
result = br.GetExtraAutotestArgs()
self.assertEqual(
result,
"--profiler=custom_perf --profiler_args='perf_options=\"record -a -e "
"cycles\"'")
self.test_benchmark.suite = 'telemetry'
result = br.GetExtraAutotestArgs()
self.assertEqual(result, '')
self.assertEqual(self.err_msg, 'Telemetry does not support profiler.')
self.test_benchmark.perf_args = 'record -e cycles'
self.test_benchmark.suite = 'test_that'
result = br.GetExtraAutotestArgs()
self.assertEqual(result, '')
self.assertEqual(self.err_msg, 'test_that does not support profiler.')
self.test_benchmark.perf_args = 'junk args'
self.test_benchmark.suite = 'telemetry_Crosperf'
self.assertRaises(Exception, br.GetExtraAutotestArgs)
def test_run_test(self, mock_result, mock_runner):
br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
self.test_label, 1,
self.test_cache_conditions,
self.mock_machine_manager,
self.mock_logger, 'average', '')
self.status = []
def MockRecord(status):
self.status.append(status)
br.timeline.Record = MockRecord
mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
'chromeos', 'average')
mock_runner.return_value = [0, "{'Score':100}", '']
br.RunTest(mock_machine)
self.assertTrue(br.run_completed)
self.assertEqual(
self.status,
[benchmark_run.STATUS_IMAGING, benchmark_run.STATUS_RUNNING])
self.assertEqual(br.machine_manager.ImageMachine.call_count, 1)
br.machine_manager.ImageMachine.assert_called_with(
mock_machine, self.test_label)
self.assertEqual(mock_runner.call_count, 1)
mock_runner.assert_called_with(mock_machine.name, br.label,
br.benchmark, '', br.profiler_args)
self.assertEqual(mock_result.call_count, 1)
mock_result.assert_called_with(self.mock_logger, 'average',
self.test_label, None, "{'Score':100}",
'', 0, 'page_cycler.netsim.top_10',
'telemetry_Crosperf')
def test_run(self):
br = benchmark_run.BenchmarkRun('test_run', self.test_benchmark,
self.test_label, 1,
self.test_cache_conditions,
self.mock_machine_manager,
self.mock_logger, 'average', '')
def MockLogOutput(msg, print_to_console=False):
'Helper function for test_run.'
del print_to_console
self.log_output.append(msg)
def MockLogError(msg, print_to_console=False):
'Helper function for test_run.'
del print_to_console
self.log_error.append(msg)
def MockRecordStatus(msg):
'Helper function for test_run.'
self.status.append(msg)
def FakeReadCache():
'Helper function for test_run.'
br.cache = mock.Mock(spec=ResultsCache)
self.called_ReadCache = True
return 0
def FakeReadCacheSucceed():
'Helper function for test_run.'
br.cache = mock.Mock(spec=ResultsCache)
br.result = mock.Mock(spec=Result)
br.result.out = 'result.out stuff'
br.result.err = 'result.err stuff'
br.result.retval = 0
self.called_ReadCache = True
return 0
def FakeReadCacheException():
'Helper function for test_run.'
raise RuntimeError(
'This is an exception test; it is supposed to happen')
def FakeAcquireMachine():
'Helper function for test_run.'
mock_machine = MockCrosMachine('chromeos1-row3-rack5-host7.cros',
'chromeos', 'average')
return mock_machine
def FakeRunTest(_machine):
'Helper function for test_run.'
mock_result = mock.Mock(spec=Result)
mock_result.retval = 0
return mock_result
def FakeRunTestFail(_machine):
'Helper function for test_run.'
mock_result = mock.Mock(spec=Result)
mock_result.retval = 1
return mock_result
def ResetTestValues():
'Helper function for test_run.'
self.log_output = []
self.log_error = []
self.status = []
br.result = None
self.called_ReadCache = False
# Assign all the fake functions to the appropriate objects.
br.logger().LogOutput = MockLogOutput
br.logger().LogError = MockLogError
br.timeline.Record = MockRecordStatus
br.ReadCache = FakeReadCache
br.RunTest = FakeRunTest
br.AcquireMachine = FakeAcquireMachine
# First test: No cache hit, all goes well.
ResetTestValues()
br.run()
self.assertTrue(self.called_ReadCache)
self.assertEqual(self.log_output, [
'test_run: No cache hit.',
'Releasing machine: chromeos1-row3-rack5-host7.cros',
'Released machine: chromeos1-row3-rack5-host7.cros'
])
self.assertEqual(len(self.log_error), 0)
self.assertEqual(self.status, ['WAITING', 'SUCCEEDED'])
# Second test: No cached result found; test run was "terminated" for some
# reason.
ResetTestValues()
br.terminated = True
br.run()
self.assertTrue(self.called_ReadCache)
self.assertEqual(self.log_output, [
'test_run: No cache hit.',
'Releasing machine: chromeos1-row3-rack5-host7.cros',
'Released machine: chromeos1-row3-rack5-host7.cros'
])
self.assertEqual(len(self.log_error), 0)
self.assertEqual(self.status, ['WAITING'])
# Third test. No cached result found; RunTest failed for some reason.
ResetTestValues()
br.terminated = False
br.RunTest = FakeRunTestFail
br.run()
self.assertTrue(self.called_ReadCache)
self.assertEqual(self.log_output, [
'test_run: No cache hit.',
'Releasing machine: chromeos1-row3-rack5-host7.cros',
'Released machine: chromeos1-row3-rack5-host7.cros'
])
self.assertEqual(len(self.log_error), 0)
self.assertEqual(self.status, ['WAITING', 'FAILED'])
# Fourth test: ReadCache found a cached result.
ResetTestValues()
br.RunTest = FakeRunTest
br.ReadCache = FakeReadCacheSucceed
br.run()
self.assertTrue(self.called_ReadCache)
self.assertEqual(self.log_output, [
'test_run: Cache hit.', 'result.out stuff',
'Releasing machine: chromeos1-row3-rack5-host7.cros',
'Released machine: chromeos1-row3-rack5-host7.cros'
])
self.assertEqual(self.log_error, ['result.err stuff'])
self.assertEqual(self.status, ['SUCCEEDED'])
# Fifth test: ReadCache generates an exception; does the try/finally block
# work?
ResetTestValues()
br.ReadCache = FakeReadCacheException
br.machine = FakeAcquireMachine()
br.run()
self.assertEqual(self.log_error, [
"Benchmark run: 'test_run' failed: This is an exception test; it is "
'supposed to happen'
])
self.assertEqual(self.status, ['FAILED'])