def run(self):
"""Executes tests.
This will instantiate controller and test classes, execute tests, and
print a summary.
This meethod should usually be called within the runner's `mobly_logger`
context. If you must use this method outside of the context, you should
make sure `self._test_run_metadata.generate_test_run_log_path` is called
before each invocation of `run`.
Raises:
Error: if no tests have previously been added to this runner using
add_test_class(...).
"""
if not self._test_run_infos:
raise Error('No tests to execute.')
# Officially starts the test run.
self._test_run_metadata.set_start_point()
# Ensure the log path exists. Necessary if `run` is used outside of the
# `mobly_logger` context.
utils.create_dir(self._test_run_metadata.root_output_path)
summary_writer = records.TestSummaryWriter(
os.path.join(self._test_run_metadata.root_output_path,
records.OUTPUT_FILE_SUMMARY))
try:
for test_run_info in self._test_run_infos:
# Set up the test-specific config
test_config = test_run_info.config.copy()
test_config.log_path = self._test_run_metadata.root_output_path
test_config.summary_writer = summary_writer
test_config.test_class_name_suffix = test_run_info.test_class_name_suffix
try:
self._run_test_class(config=test_config,
test_class=test_run_info.test_class,
tests=test_run_info.tests)
except signals.TestAbortAll as e:
logging.warning(
'Abort all subsequent test classes. Reason: %s', e)
raise
finally:
summary_writer.dump(self.results.summary_dict(),
records.TestSummaryEntryType.SUMMARY)
self._test_run_metadata.set_end_point()
# Show the test run summary.
summary_lines = [
f'Summary for test run {self._test_run_metadata.run_id}:',
f'Total time elapsed {self._test_run_metadata.time_elapsed_sec}s',
f'Artifacts are saved in "{self._test_run_metadata.root_output_path}"',
f'Test results: {self.results.summary_str()}'
]
logging.info('\n'.join(summary_lines))
def test_summary_write_dump(self):
s = signals.TestFailure(self.details, self.float_extra)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(s)
dump_path = os.path.join(self.tmp_path, 'ha.yaml')
writer = records.TestSummaryWriter(dump_path)
writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD)
with open(dump_path, 'r') as f:
content = yaml.load(f)
self.assertEqual(content['Type'],
records.TestSummaryEntryType.RECORD.value)
def test_summary_write_dump(self):
s = signals.TestFailure(self.details, self.float_extra)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(s)
dump_path = os.path.join(self.tmp_path, 'ha.yaml')
writer = records.TestSummaryWriter(dump_path)
writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD)
with io.open(dump_path, 'r', encoding='utf-8') as f:
content = yaml.safe_load(f)
self.assertEqual(content['Type'],
records.TestSummaryEntryType.RECORD.value)
self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS],
self.details)
self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS],
self.float_extra)
def run(self):
"""Executes tests.
This will instantiate controller and test classes, execute tests, and
print a summary.
Raises:
Error: if no tests have previously been added to this runner using
add_test_class(...).
"""
if not self._test_run_infos:
raise Error('No tests to execute.')
self.setup_logger()
summary_writer = records.TestSummaryWriter(
os.path.join(self._log_path, records.OUTPUT_FILE_SUMMARY))
try:
for test_run_info in self._test_run_infos:
# Set up the test-specific config
test_config = test_run_info.config.copy()
test_config.log_path = self._log_path
test_config.register_controller = functools.partial(
self._register_controller, test_config)
test_config.summary_writer = summary_writer
test_config.test_class_name_suffix = test_run_info.test_class_name_suffix
try:
self._run_test_class(config=test_config,
test_class=test_run_info.test_class,
tests=test_run_info.tests)
except signals.TestAbortAll as e:
logging.warning(
'Abort all subsequent test classes. Reason: %s', e)
raise
finally:
self._unregister_controllers()
finally:
# Write controller info and summary to summary file.
summary_writer.dump(self.results.controller_info,
records.TestSummaryEntryType.CONTROLLER_INFO)
summary_writer.dump(self.results.summary_dict(),
records.TestSummaryEntryType.SUMMARY)
# Stop and show summary.
msg = '\nSummary for test run %s@%s: %s\n' % (
self._test_bed_name, self._start_time,
self.results.summary_str())
logging.info(msg.strip())
self._teardown_logger()
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.mock_test_cls_configs = config_parser.TestRunConfig()
self.summary_file = os.path.join(self.tmp_dir, 'summary.yaml')
self.mock_test_cls_configs.summary_writer = records.TestSummaryWriter(
self.summary_file)
self.mock_test_cls_configs.log_path = self.tmp_dir
self.mock_test_cls_configs.user_params = {"some_param": "hahaha"}
self.mock_test_cls_configs.reporter = mock.MagicMock()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
def test_summary_user_data(self):
user_data1 = {'a': 1}
user_data2 = {'b': 1}
user_data = [user_data1, user_data2]
dump_path = os.path.join(self.tmp_path, 'ha.yaml')
writer = records.TestSummaryWriter(dump_path)
for data in user_data:
writer.dump(data, records.TestSummaryEntryType.USER_DATA)
with open(dump_path, 'r') as f:
contents = []
for c in yaml.load_all(f):
contents.append(c)
for content in contents:
self.assertEqual(content['Type'],
records.TestSummaryEntryType.USER_DATA.value)
self.assertEqual(contents[0]['a'], user_data1['a'])
self.assertEqual(contents[1]['b'], user_data2['b'])
def test_summary_write_dump_with_unicode(self):
unicode_details = u'\u901a' # utf-8 -> b'\xe9\x80\x9a'
unicode_extras = u'\u8fc7' # utf-8 -> b'\xe8\xbf\x87'
s = signals.TestFailure(unicode_details, unicode_extras)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(s)
dump_path = os.path.join(self.tmp_path, 'ha.yaml')
writer = records.TestSummaryWriter(dump_path)
writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD)
with io.open(dump_path, 'r', encoding='utf-8') as f:
content = yaml.load(f)
self.assertEqual(content['Type'],
records.TestSummaryEntryType.RECORD.value)
self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS],
unicode_details)
self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS],
unicode_extras)
def run(self):
"""Executes tests.
This will instantiate controller and test classes, execute tests, and
print a summary.
Raises:
Error: if no tests have previously been added to this runner using
add_test_class(...).
"""
if not self._test_run_infos:
raise Error('No tests to execute.')
# Ensure the log path exists. Necessary if `run` is used outside of the
# `mobly_logger` context.
utils.create_dir(self._log_path)
summary_writer = records.TestSummaryWriter(
os.path.join(self._log_path, records.OUTPUT_FILE_SUMMARY))
try:
for test_run_info in self._test_run_infos:
# Set up the test-specific config
test_config = test_run_info.config.copy()
test_config.log_path = self._log_path
test_config.summary_writer = summary_writer
test_config.test_class_name_suffix = test_run_info.test_class_name_suffix
try:
self._run_test_class(config=test_config,
test_class=test_run_info.test_class,
tests=test_run_info.tests)
except signals.TestAbortAll as e:
logging.warning(
'Abort all subsequent test classes. Reason: %s', e)
raise
finally:
summary_writer.dump(self.results.summary_dict(),
records.TestSummaryEntryType.SUMMARY)
# Stop and show summary.
msg = '\nSummary for test run %s@%s: %s\n' % (
self._test_bed_name, self._start_time,
self.results.summary_str())
logging.info(msg.strip())
def test_mobly_test(self, mock_set, mock_create_device):
"""Use a fake mobly test to verify everything actually works as expected."""
class FakeTest(base_test.BaseTestClass):
"""Fake Mobly test to verify functionality."""
def setup_class(self):
self.devices = self.register_controller(gazoo_device)
def test_1(self):
asserts.assert_true(len(self.devices) == 2, self.devices)
fake_config = config_parser.TestRunConfig()
fake_config.log_path = self.artifacts_directory
summary_file = os.path.join(self.artifacts_directory, "summary.yaml")
fake_config.summary_writer = records.TestSummaryWriter(summary_file)
fake_config.controller_configs = FAKE_CONTROLLER_CONFIGS
test = FakeTest(fake_config)
test.run(["test_1"])
self.assertTrue(test.results.passed,
"Test results: {}".format(test.results))
actual_record = test.results.passed[0]
self.assertIn(actual_record.test_name, "test_1")
