def test_init(self):
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
self.assertFalse(er._terminated)
self.assertEqual(er.STATUS_TIME_DELAY, 10)
self.exp.log_level = 'verbose'
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
self.assertEqual(er.STATUS_TIME_DELAY, 30)
def test_store_results(self, mock_cleanup, mock_copy, _mock_text_report,
mock_report, mock_writefile, mock_mkdir,
mock_rmdir):
self.mock_logger.Reset()
self.exp.results_directory = '/usr/local/crosperf-results'
bench_run = self.exp.benchmark_runs[5]
bench_path = '/usr/local/crosperf-results/' + filter(
str.isalnum, bench_run.name)
self.assertEqual(len(self.exp.benchmark_runs), 6)
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
# Test 1. Make sure nothing is done if _terminated is true.
er._terminated = True
er._StoreResults(self.exp)
self.assertEqual(mock_cleanup.call_count, 0)
self.assertEqual(mock_copy.call_count, 0)
self.assertEqual(mock_report.call_count, 0)
self.assertEqual(mock_writefile.call_count, 0)
self.assertEqual(mock_mkdir.call_count, 0)
self.assertEqual(mock_rmdir.call_count, 0)
self.assertEqual(self.mock_logger.LogOutputCount, 0)
# Test 2. _terminated is false; everything works properly.
fake_result = Result(self.mock_logger, self.exp.labels[0], 'average',
'daisy1')
for r in self.exp.benchmark_runs:
r.result = fake_result
er._terminated = False
er._StoreResults(self.exp)
self.assertEqual(mock_cleanup.call_count, 6)
mock_cleanup.called_with(bench_run.benchmark.rm_chroot_tmp)
self.assertEqual(mock_copy.call_count, 6)
mock_copy.called_with(bench_path)
self.assertEqual(mock_writefile.call_count, 3)
self.assertEqual(len(mock_writefile.call_args_list), 3)
first_args = mock_writefile.call_args_list[0]
second_args = mock_writefile.call_args_list[1]
self.assertEqual(first_args[0][0],
'/usr/local/crosperf-results/experiment.exp')
self.assertEqual(second_args[0][0],
'/usr/local/crosperf-results/results.html')
self.assertEqual(mock_mkdir.call_count, 1)
mock_mkdir.called_with('/usr/local/crosperf-results')
self.assertEqual(mock_rmdir.call_count, 1)
mock_rmdir.called_with('/usr/local/crosperf-results')
self.assertEqual(self.mock_logger.LogOutputCount, 4)
self.assertEqual(self.mock_logger.output_msgs, [
'Storing experiment file in /usr/local/crosperf-results.',
'Storing results report in /usr/local/crosperf-results.',
'Storing email message body in /usr/local/crosperf-results.',
'Storing results of each benchmark run.'
])
def test_print_table(self, mock_report):
self.mock_logger.Reset()
mock_report.return_value = 'This is a fake experiment report.'
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
er._PrintTable(self.exp)
self.assertEqual(mock_report.call_count, 1)
self.assertEqual(self.mock_logger.output_msgs,
['This is a fake experiment report.'])
def test_email(self, mock_getuser, mock_emailer, mock_attachment,
mock_text_report, mock_html_report):
mock_getuser.return_value = '*****@*****.**'
mock_text_report.return_value = 'This is a fake text report.'
mock_html_report.return_value = 'This is a fake html report.'
self.mock_logger.Reset()
config.AddConfig('no_email', True)
self.exp.email_to = ['*****@*****.**']
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
# Test 1. Config:no_email; exp.email_to set ==> no email sent
er._Email(self.exp)
self.assertEqual(mock_getuser.call_count, 0)
self.assertEqual(mock_emailer.call_count, 0)
self.assertEqual(mock_attachment.call_count, 0)
self.assertEqual(mock_text_report.call_count, 0)
self.assertEqual(mock_html_report.call_count, 0)
# Test 2. Config: email. exp.email_to set; cache hit. => send email
self.mock_logger.Reset()
config.AddConfig('no_email', False)
for r in self.exp.benchmark_runs:
r.cache_hit = True
er._Email(self.exp)
self.assertEqual(mock_getuser.call_count, 1)
self.assertEqual(mock_emailer.call_count, 1)
self.assertEqual(mock_attachment.call_count, 1)
self.assertEqual(mock_text_report.call_count, 1)
self.assertEqual(mock_html_report.call_count, 1)
self.assertEqual(len(mock_emailer.call_args), 2)
self.assertEqual(mock_emailer.call_args[0],
(['*****@*****.**', '*****@*****.**'],
': image1 vs. image2',
"<pre style='font-size: 13px'>This is a fake text "
'report.\nResults are stored in _results.\n</pre>'))
self.assertTrue(type(mock_emailer.call_args[1]) is dict)
self.assertEqual(len(mock_emailer.call_args[1]), 2)
self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
mock_attachment.assert_called_with('report.html',
'This is a fake html report.')
# Test 3. Config: email; exp.mail_to set; no cache hit. => send email
self.mock_logger.Reset()
mock_getuser.reset_mock()
mock_emailer.reset_mock()
mock_attachment.reset_mock()
mock_text_report.reset_mock()
mock_html_report.reset_mock()
config.AddConfig('no_email', False)
for r in self.exp.benchmark_runs:
r.cache_hit = False
er._Email(self.exp)
self.assertEqual(mock_getuser.call_count, 1)
self.assertEqual(mock_emailer.call_count, 1)
self.assertEqual(mock_attachment.call_count, 1)
self.assertEqual(mock_text_report.call_count, 1)
self.assertEqual(mock_html_report.call_count, 1)
self.assertEqual(len(mock_emailer.call_args), 2)
self.assertEqual(mock_emailer.call_args[0],
(['*****@*****.**', '*****@*****.**',
'*****@*****.**'], ': image1 vs. image2',
"<pre style='font-size: 13px'>This is a fake text "
'report.\nResults are stored in _results.\n</pre>'))
self.assertTrue(type(mock_emailer.call_args[1]) is dict)
self.assertEqual(len(mock_emailer.call_args[1]), 2)
self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
mock_attachment.assert_called_with('report.html',
'This is a fake html report.')
# Test 4. Config: email; exp.mail_to = None; no cache hit. => send email
self.mock_logger.Reset()
mock_getuser.reset_mock()
mock_emailer.reset_mock()
mock_attachment.reset_mock()
mock_text_report.reset_mock()
mock_html_report.reset_mock()
self.exp.email_to = []
er._Email(self.exp)
self.assertEqual(mock_getuser.call_count, 1)
self.assertEqual(mock_emailer.call_count, 1)
self.assertEqual(mock_attachment.call_count, 1)
self.assertEqual(mock_text_report.call_count, 1)
self.assertEqual(mock_html_report.call_count, 1)
self.assertEqual(len(mock_emailer.call_args), 2)
self.assertEqual(mock_emailer.call_args[0],
(['*****@*****.**'], ': image1 vs. image2',
"<pre style='font-size: 13px'>This is a fake text "
'report.\nResults are stored in _results.\n</pre>'))
self.assertTrue(type(mock_emailer.call_args[1]) is dict)
self.assertEqual(len(mock_emailer.call_args[1]), 2)
self.assertTrue('attachments' in mock_emailer.call_args[1].keys())
self.assertEqual(mock_emailer.call_args[1]['msg_type'], 'html')
mock_attachment.assert_called_with('report.html',
'This is a fake html report.')
# Test 5. Config: email; exp.mail_to = None; cache hit => no email sent
self.mock_logger.Reset()
mock_getuser.reset_mock()
mock_emailer.reset_mock()
mock_attachment.reset_mock()
mock_text_report.reset_mock()
mock_html_report.reset_mock()
for r in self.exp.benchmark_runs:
r.cache_hit = True
er._Email(self.exp)
self.assertEqual(mock_getuser.call_count, 0)
self.assertEqual(mock_emailer.call_count, 0)
self.assertEqual(mock_attachment.call_count, 0)
self.assertEqual(mock_text_report.call_count, 0)
self.assertEqual(mock_html_report.call_count, 0)
def test_run(self, mock_progress_string, mock_status_string):
self.run_count = 0
self.is_complete_count = 0
def reset():
self.run_count = 0
self.is_complete_count = 0
def FakeRun():
self.run_count += 1
return 0
def FakeIsComplete():
self.is_complete_count += 1
if self.is_complete_count < 3:
return False
else:
return True
self.mock_logger.Reset()
self.exp.Run = FakeRun
self.exp.IsComplete = FakeIsComplete
# Test 1: log_level == "quiet"
self.exp.log_level = 'quiet'
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
er.STATUS_TIME_DELAY = 2
mock_status_string.return_value = 'Fake status string'
er._Run(self.exp)
self.assertEqual(self.run_count, 1)
self.assertTrue(self.is_complete_count > 0)
self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
self.assertEqual(self.mock_logger.dot_count, 2)
self.assertEqual(mock_progress_string.call_count, 0)
self.assertEqual(mock_status_string.call_count, 2)
self.assertEqual(self.mock_logger.output_msgs,
['==============================', 'Fake status string',
'=============================='])
self.assertEqual(len(self.mock_logger.error_msgs), 0)
# Test 2: log_level == "average"
self.mock_logger.Reset()
reset()
self.exp.log_level = 'average'
mock_status_string.call_count = 0
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
er.STATUS_TIME_DELAY = 2
mock_status_string.return_value = 'Fake status string'
er._Run(self.exp)
self.assertEqual(self.run_count, 1)
self.assertTrue(self.is_complete_count > 0)
self.assertEqual(self.mock_logger.LogStartDotsCount, 1)
self.assertEqual(self.mock_logger.LogAppendDotCount, 1)
self.assertEqual(self.mock_logger.LogEndDotsCount, 1)
self.assertEqual(self.mock_logger.dot_count, 2)
self.assertEqual(mock_progress_string.call_count, 0)
self.assertEqual(mock_status_string.call_count, 2)
self.assertEqual(self.mock_logger.output_msgs,
['==============================', 'Fake status string',
'=============================='])
self.assertEqual(len(self.mock_logger.error_msgs), 0)
# Test 3: log_level == "verbose"
self.mock_logger.Reset()
reset()
self.exp.log_level = 'verbose'
mock_status_string.call_count = 0
er = experiment_runner.ExperimentRunner(self.exp,
json_report=False,
using_schedv2=False,
log=self.mock_logger,
cmd_exec=self.mock_cmd_exec)
er.STATUS_TIME_DELAY = 2
mock_status_string.return_value = 'Fake status string'
mock_progress_string.return_value = 'Fake progress string'
er._Run(self.exp)
self.assertEqual(self.run_count, 1)
self.assertTrue(self.is_complete_count > 0)
self.assertEqual(self.mock_logger.LogStartDotsCount, 0)
self.assertEqual(self.mock_logger.LogAppendDotCount, 0)
self.assertEqual(self.mock_logger.LogEndDotsCount, 0)
self.assertEqual(self.mock_logger.dot_count, 0)
self.assertEqual(mock_progress_string.call_count, 2)
self.assertEqual(mock_status_string.call_count, 2)
self.assertEqual(self.mock_logger.output_msgs,
['==============================', 'Fake progress string',
'Fake status string', '==============================',
'==============================', 'Fake progress string',
'Fake status string', '=============================='])
self.assertEqual(len(self.mock_logger.error_msgs), 0)
