def run(self): """Executes tests. This will instantiate controller and test classes, execute tests, and print a summary. This meethod should usually be called within the runner's `mobly_logger` context. If you must use this method outside of the context, you should make sure `self._test_run_metadata.generate_test_run_log_path` is called before each invocation of `run`. Raises: Error: if no tests have previously been added to this runner using add_test_class(...). """ if not self._test_run_infos: raise Error('No tests to execute.') # Officially starts the test run. self._test_run_metadata.set_start_point() # Ensure the log path exists. Necessary if `run` is used outside of the # `mobly_logger` context. utils.create_dir(self._test_run_metadata.root_output_path) summary_writer = records.TestSummaryWriter( os.path.join(self._test_run_metadata.root_output_path, records.OUTPUT_FILE_SUMMARY)) try: for test_run_info in self._test_run_infos: # Set up the test-specific config test_config = test_run_info.config.copy() test_config.log_path = self._test_run_metadata.root_output_path test_config.summary_writer = summary_writer test_config.test_class_name_suffix = test_run_info.test_class_name_suffix try: self._run_test_class(config=test_config, test_class=test_run_info.test_class, tests=test_run_info.tests) except signals.TestAbortAll as e: logging.warning( 'Abort all subsequent test classes. Reason: %s', e) raise finally: summary_writer.dump(self.results.summary_dict(), records.TestSummaryEntryType.SUMMARY) self._test_run_metadata.set_end_point() # Show the test run summary. summary_lines = [ f'Summary for test run {self._test_run_metadata.run_id}:', f'Total time elapsed {self._test_run_metadata.time_elapsed_sec}s', f'Artifacts are saved in "{self._test_run_metadata.root_output_path}"', f'Test results: {self.results.summary_str()}' ] logging.info('\n'.join(summary_lines))
def test_summary_write_dump(self): s = signals.TestFailure(self.details, self.float_extra) record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(s) dump_path = os.path.join(self.tmp_path, 'ha.yaml') writer = records.TestSummaryWriter(dump_path) writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD) with open(dump_path, 'r') as f: content = yaml.load(f) self.assertEqual(content['Type'], records.TestSummaryEntryType.RECORD.value)
def test_summary_write_dump(self): s = signals.TestFailure(self.details, self.float_extra) record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(s) dump_path = os.path.join(self.tmp_path, 'ha.yaml') writer = records.TestSummaryWriter(dump_path) writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD) with io.open(dump_path, 'r', encoding='utf-8') as f: content = yaml.safe_load(f) self.assertEqual(content['Type'], records.TestSummaryEntryType.RECORD.value) self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS], self.details) self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS], self.float_extra)
def run(self): """Executes tests. This will instantiate controller and test classes, execute tests, and print a summary. Raises: Error: if no tests have previously been added to this runner using add_test_class(...). """ if not self._test_run_infos: raise Error('No tests to execute.') self.setup_logger() summary_writer = records.TestSummaryWriter( os.path.join(self._log_path, records.OUTPUT_FILE_SUMMARY)) try: for test_run_info in self._test_run_infos: # Set up the test-specific config test_config = test_run_info.config.copy() test_config.log_path = self._log_path test_config.register_controller = functools.partial( self._register_controller, test_config) test_config.summary_writer = summary_writer test_config.test_class_name_suffix = test_run_info.test_class_name_suffix try: self._run_test_class(config=test_config, test_class=test_run_info.test_class, tests=test_run_info.tests) except signals.TestAbortAll as e: logging.warning( 'Abort all subsequent test classes. Reason: %s', e) raise finally: self._unregister_controllers() finally: # Write controller info and summary to summary file. summary_writer.dump(self.results.controller_info, records.TestSummaryEntryType.CONTROLLER_INFO) summary_writer.dump(self.results.summary_dict(), records.TestSummaryEntryType.SUMMARY) # Stop and show summary. msg = '\nSummary for test run %s@%s: %s\n' % ( self._test_bed_name, self._start_time, self.results.summary_str()) logging.info(msg.strip()) self._teardown_logger()
def setUp(self): self.tmp_dir = tempfile.mkdtemp() self.mock_test_cls_configs = config_parser.TestRunConfig() self.summary_file = os.path.join(self.tmp_dir, 'summary.yaml') self.mock_test_cls_configs.summary_writer = records.TestSummaryWriter( self.summary_file) self.mock_test_cls_configs.log_path = self.tmp_dir self.mock_test_cls_configs.user_params = {"some_param": "hahaha"} self.mock_test_cls_configs.reporter = mock.MagicMock() self.base_mock_test_config = config_parser.TestRunConfig() self.base_mock_test_config.test_bed_name = 'SampleTestBed' self.base_mock_test_config.controller_configs = {} self.base_mock_test_config.user_params = { 'icecream': 42, 'extra_param': 'haha' } self.base_mock_test_config.log_path = self.tmp_dir
def test_summary_user_data(self): user_data1 = {'a': 1} user_data2 = {'b': 1} user_data = [user_data1, user_data2] dump_path = os.path.join(self.tmp_path, 'ha.yaml') writer = records.TestSummaryWriter(dump_path) for data in user_data: writer.dump(data, records.TestSummaryEntryType.USER_DATA) with open(dump_path, 'r') as f: contents = [] for c in yaml.load_all(f): contents.append(c) for content in contents: self.assertEqual(content['Type'], records.TestSummaryEntryType.USER_DATA.value) self.assertEqual(contents[0]['a'], user_data1['a']) self.assertEqual(contents[1]['b'], user_data2['b'])
def test_summary_write_dump_with_unicode(self): unicode_details = u'\u901a' # utf-8 -> b'\xe9\x80\x9a' unicode_extras = u'\u8fc7' # utf-8 -> b'\xe8\xbf\x87' s = signals.TestFailure(unicode_details, unicode_extras) record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(s) dump_path = os.path.join(self.tmp_path, 'ha.yaml') writer = records.TestSummaryWriter(dump_path) writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD) with io.open(dump_path, 'r', encoding='utf-8') as f: content = yaml.load(f) self.assertEqual(content['Type'], records.TestSummaryEntryType.RECORD.value) self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS], unicode_details) self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS], unicode_extras)
def run(self): """Executes tests. This will instantiate controller and test classes, execute tests, and print a summary. Raises: Error: if no tests have previously been added to this runner using add_test_class(...). """ if not self._test_run_infos: raise Error('No tests to execute.') # Ensure the log path exists. Necessary if `run` is used outside of the # `mobly_logger` context. utils.create_dir(self._log_path) summary_writer = records.TestSummaryWriter( os.path.join(self._log_path, records.OUTPUT_FILE_SUMMARY)) try: for test_run_info in self._test_run_infos: # Set up the test-specific config test_config = test_run_info.config.copy() test_config.log_path = self._log_path test_config.summary_writer = summary_writer test_config.test_class_name_suffix = test_run_info.test_class_name_suffix try: self._run_test_class(config=test_config, test_class=test_run_info.test_class, tests=test_run_info.tests) except signals.TestAbortAll as e: logging.warning( 'Abort all subsequent test classes. Reason: %s', e) raise finally: summary_writer.dump(self.results.summary_dict(), records.TestSummaryEntryType.SUMMARY) # Stop and show summary. msg = '\nSummary for test run %s@%s: %s\n' % ( self._test_bed_name, self._start_time, self.results.summary_str()) logging.info(msg.strip())
def test_mobly_test(self, mock_set, mock_create_device): """Use a fake mobly test to verify everything actually works as expected.""" class FakeTest(base_test.BaseTestClass): """Fake Mobly test to verify functionality.""" def setup_class(self): self.devices = self.register_controller(gazoo_device) def test_1(self): asserts.assert_true(len(self.devices) == 2, self.devices) fake_config = config_parser.TestRunConfig() fake_config.log_path = self.artifacts_directory summary_file = os.path.join(self.artifacts_directory, "summary.yaml") fake_config.summary_writer = records.TestSummaryWriter(summary_file) fake_config.controller_configs = FAKE_CONTROLLER_CONFIGS test = FakeTest(fake_config) test.run(["test_1"]) self.assertTrue(test.results.passed, "Test results: {}".format(test.results)) actual_record = test.results.passed[0] self.assertIn(actual_record.test_name, "test_1")