def test_basic_output(self):
"""Verifies the expected output files from a test run.
* Files are correctly created.
* Basic sanity checks of each output file.
"""
mock_test_config = self.create_mock_test_config(
self.base_mock_test_config)
tr = test_runner.TestRunner(self.log_dir, self.testbed_name)
with tr.mobly_logger():
tr.add_test_class(mock_test_config,
integration_test.IntegrationTest)
tr.run()
expected_class_path = os.path.join(logging.root_output_path,
'IntegrationTest')
self.assertEqual(expected_class_path, logging.log_path)
os.path.exists(logging.log_path)
output_dir = logging.root_output_path
(summary_file_path, debug_log_path,
info_log_path) = self.assert_output_logs_exist(output_dir)
summary_entries = []
with io.open(summary_file_path, 'r', encoding='utf-8') as f:
for entry in yaml.safe_load_all(f):
self.assertTrue(entry['Type'])
summary_entries.append(entry)
self.assert_log_contents(debug_log_path, whitelist=['DEBUG', 'INFO'])
self.assert_log_contents(info_log_path,
whitelist=['INFO'],
blacklist=['DEBUG'])
def test_logging_before_run(self):
"""Verifies the expected output files from a test run.
* Files are correctly created.
* Basic sanity checks of each output file.
"""
mock_test_config = self.create_mock_test_config(
self.base_mock_test_config)
info_uuid = 'e098d4ff-4e90-4e08-b369-aa84a7ef90ec'
debug_uuid = 'c6f1474e-960a-4df8-8305-1c5b8b905eca'
tr = test_runner.TestRunner(self.log_dir, self.testbed_name)
with tr.mobly_logger():
logging.info(info_uuid)
logging.debug(debug_uuid)
tr.add_test_class(mock_test_config,
integration_test.IntegrationTest)
tr.run()
output_dir = logging.root_output_path
(summary_file_path, debug_log_path,
info_log_path) = self.assert_output_logs_exist(output_dir)
self.assert_log_contents(debug_log_path,
whitelist=[debug_uuid, info_uuid])
self.assert_log_contents(info_log_path,
whitelist=[info_uuid],
blacklist=[debug_uuid])
def test_teardown_class_output(self):
"""Verifies the summary file includes the failure record for
teardown_class.
"""
mock_test_config = self.base_mock_test_config.copy()
tr = test_runner.TestRunner(self.log_dir, self.testbed_name)
with tr.mobly_logger():
tr.add_test_class(
mock_test_config,
teardown_class_failure_test.TearDownClassFailureTest)
tr.run()
output_dir = logging.root_output_path
summary_file_path = os.path.join(output_dir,
records.OUTPUT_FILE_SUMMARY)
found = False
with io.open(summary_file_path, 'r', encoding='utf-8') as f:
raw_content = f.read()
f.seek(0)
for entry in yaml.safe_load_all(f):
if (entry['Type'] == 'Record'
and entry[records.TestResultEnums.RECORD_NAME]
== 'teardown_class'):
found = True
break
self.assertTrue(
found,
'No record for teardown_class found in the output file:\n %s' %
raw_content)
def test_register_controller_builtin_dup_register(self):
"""Same as test_register_controller_third_party_dup_register, except
this is for a builtin controller module.
"""
mock_test_config = dict(self.base_mock_test_config)
tb_key = keys.Config.key_testbed.value
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_ref_name = "haha"
setattr(mock_controller, "MOBLY_CONTROLLER_REFERENCE_NAME",
mock_ref_name)
try:
mock_ctrlr_ref_name = mock_controller.MOBLY_CONTROLLER_REFERENCE_NAME
mock_test_config[tb_key][mock_ctrlr_config_name] = [
"magic1", "magic2"
]
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
tr.register_controller(mock_controller)
self.assertTrue(mock_ref_name in tr.test_run_info)
self.assertTrue(mock_ref_name in tr.controller_registry)
mock_ctrlrs = tr.test_run_info[mock_ctrlr_ref_name]
self.assertEqual(mock_ctrlrs[0].magic, "magic1")
self.assertEqual(mock_ctrlrs[1].magic, "magic2")
self.assertTrue(tr.controller_destructors[mock_ctrlr_ref_name])
expected_msg = "Controller module .* has already been registered."
with self.assertRaisesRegexp(signals.ControllerError,
expected_msg):
tr.register_controller(mock_controller)
finally:
delattr(mock_controller, "MOBLY_CONTROLLER_REFERENCE_NAME")
def test_summary_file_entries(self):
"""Verifies the output summary's file format.
This focuses on the format of the file instead of the content of
entries, which is covered in base_test_test.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
summary_path = os.path.join(mock_test_config.log_path,
mock_test_config.test_bed_name, 'latest',
records.OUTPUT_FILE_SUMMARY)
with open(summary_path, 'r') as f:
summary_entries = list(yaml.load_all(f))
self.assertEqual(len(summary_entries), 4)
# Verify the first entry is the list of test names.
self.assertEqual(summary_entries[0]['Type'],
records.TestSummaryEntryType.TEST_NAME_LIST.value)
self.assertEqual(summary_entries[1]['Type'],
records.TestSummaryEntryType.RECORD.value)
def test_run_two_test_classes(self, mock_get_all, mock_list_adb,
mock_fastboot, mock_adb):
"""Verifies that running more than one test class in one test run works
properly.
This requires using a built-in controller module. Using AndroidDevice
module since it has all the mocks needed already.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
mock_test_config.controller_configs['AndroidDevice'] = [{
'serial': '1'
}]
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration2_test.Integration2Test)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
self.assertFalse(tr._controller_registry)
self.assertFalse(tr._controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
def test_teardown_logger_before_setup_logger(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\._teardown_logger\(\) called before '
'TestRunner\.setup_logger\(\)!'):
tr._teardown_logger()
def test_run_two_test_classes_different_configs_and_aliases(self):
"""Verifies that running more than one test class in one test run with
different configs works properly.
"""
config1 = self.base_mock_test_config.copy()
config1.controller_configs[
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME] = [{
'serial': 'xxxx'
}]
config2 = config1.copy()
config2.user_params['icecream'] = 10
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(config1,
integration_test.IntegrationTest,
name_suffix='FirstConfig')
tr.add_test_class(config2,
integration_test.IntegrationTest,
name_suffix='SecondConfig')
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 1)
self.assertEqual(results['Failed'], 1)
self.assertEqual(tr.results.failed[0].details, '10 != 42')
record1 = tr.results.executed[0]
record2 = tr.results.executed[1]
self.assertEqual(record1.test_class, 'IntegrationTest_FirstConfig')
self.assertEqual(record2.test_class, 'IntegrationTest_SecondConfig')
def run_suite_class(argv=None):
"""Executes tests in the test suite.
Args:
argv: A list that is then parsed as CLI args. If None, defaults to sys.argv.
"""
cli_args = _parse_cli_args(argv)
test_configs = config_parser.load_test_config_file(cli_args.config)
config_count = len(test_configs)
if config_count != 1:
logging.error('Expect exactly one test config, found %d', config_count)
config = test_configs[0]
runner = test_runner.TestRunner(log_dir=config.log_path,
testbed_name=config.testbed_name)
suite_class = _find_suite_class()
suite = suite_class(runner, config)
ok = False
with runner.mobly_logger():
try:
suite.setup_suite(config.copy())
try:
runner.run()
ok = runner.results.is_all_pass
print(ok)
except signals.TestAbortAll:
pass
finally:
suite.teardown_suite()
if not ok:
sys.exit(1)
def test_symlink(self):
"""Verifies the symlink is created and links properly."""
mock_test_config = self.create_mock_test_config(
self.base_mock_test_config)
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.setup_logger()
symlink = os.path.join(self.log_dir, self.test_bed_name, 'latest')
self.assertEqual(os.readlink(symlink), logging.log_path)
def test_mobly_logger_skips_latest_log_alias_when_empty(
self, mock_create_alias):
mock_test_config = self.create_mock_test_config(
self.base_mock_test_config)
tr = test_runner.TestRunner(self.log_dir, self.testbed_name)
with tr.mobly_logger(alias=''):
pass
mock_create_alias.asset_not_called()
def test_add_test_class_mismatched_log_path(self):
tr = test_runner.TestRunner('/different/log/dir', self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s log folder is "/different/log/dir", but a test '
r'config with a different log folder \("%s"\) was added.' %
self.log_dir):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_register_controller_return_value(self):
mock_test_config = dict(self.base_mock_test_config)
tb_key = keys.Config.key_testbed.value
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config[tb_key][mock_ctrlr_config_name] = ["magic1", "magic2"]
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
magic_devices = tr.register_controller(mock_controller)
self.assertEqual(magic_devices[0].magic, "magic1")
self.assertEqual(magic_devices[1].magic, "magic2")
def test_add_test_class_mismatched_test_bed_name(self):
tr = test_runner.TestRunner(self.log_dir, 'different_test_bed')
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s test bed is "different_test_bed", but a test '
r'config with a different test bed \("%s"\) was added.' %
self.test_bed_name):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_register_controller_less_than_min_number(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
expected_msg = 'Expected to get at least 3 controller objects, got 2.'
with self.assertRaisesRegexp(signals.ControllerError, expected_msg):
tr.register_controller(mock_controller, min_number=3)
def test_register_controller_return_value(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
magic_devices = tr.register_controller(mock_controller)
self.assertEqual(magic_devices[0].magic, 'magic1')
self.assertEqual(magic_devices[1].magic, 'magic2')
def test_mobly_logger_with_custom_latest_log_alias(self,
mock_create_alias):
mock_test_config = self.create_mock_test_config(
self.base_mock_test_config)
tr = test_runner.TestRunner(self.log_dir, self.testbed_name)
with tr.mobly_logger(alias='history'):
pass
expected_alias_dir = os.path.join(self.log_dir, self.testbed_name,
'history')
mock_create_alias.assert_called_once_with(logging.log_path,
expected_alias_dir)
def test_run_with_abort_all(self):
mock_test_config = self.base_mock_test_config.copy()
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration3_test.Integration3Test)
with self.assertRaises(signals.TestAbortAll):
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 1)
self.assertEqual(results['Executed'], 0)
self.assertEqual(results['Passed'], 0)
self.assertEqual(results['Failed'], 0)
def test_register_controller_change_return_value(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
magic_devices = tr._register_controller(mock_test_config,
mock_controller)
magic1 = magic_devices.pop(0)
self.assertIs(magic1, tr._controller_registry['mock_controller'][0])
self.assertEqual(len(tr._controller_registry['mock_controller']), 2)
def test_register_optional_controller_third_party_dup_register(self):
"""Verifies correctness of registration, internal tally of controllers
objects, and the right error happen when an optional controller module
is registered twice.
"""
mock_test_config = dict(self.base_mock_test_config)
tb_key = keys.Config.key_testbed.value
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config[tb_key][mock_ctrlr_config_name] = ["magic1", "magic2"]
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
tr.register_controller(mock_controller, required=False)
expected_msg = "Controller module .* has already been registered."
with self.assertRaisesRegexp(signals.ControllerError, expected_msg):
tr.register_controller(mock_controller, required=False)
def test_run_without_mobly_logger_context(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
self.base_mock_test_config.controller_configs[
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME] = '*'
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 1)
self.assertEqual(results['Executed'], 1)
self.assertEqual(results['Passed'], 1)
self.assertEqual(len(tr.results.executed), 1)
record = tr.results.executed[0]
self.assertEqual(record.test_class, 'IntegrationTest')
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with tr.mobly_logger():
tr.add_test_class(mock_test_config,
integration_test.IntegrationTest)
tr.run()
self.assertTrue(
mock_test_config.controller_configs[mock_ctrlr_config_name][0])
with tr.mobly_logger():
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
expected_info_dict = {
'Controller Info': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}],
'Controller Name':
'MagicDevice',
'Test Class':
'IntegrationTest',
}
self._assertControllerInfoEqual(tr.results.controller_info[0],
expected_info_dict)
self._assertControllerInfoEqual(tr.results.controller_info[1],
expected_info_dict)
self.assertNotEqual(tr.results.controller_info[0],
tr.results.controller_info[1])
def test_register_controller_no_get_info(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
get_info = getattr(mock_controller, 'get_info')
delattr(mock_controller, 'get_info')
try:
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr._register_controller(mock_test_config, mock_controller)
self.assertEqual(tr.results.controller_info, {})
finally:
setattr(mock_controller, 'get_info', get_info)
def test_register_controller_no_get_info(self):
mock_test_config = dict(self.base_mock_test_config)
tb_key = keys.Config.key_testbed.value
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_ref_name = "haha"
get_info = getattr(mock_controller, "get_info")
delattr(mock_controller, "get_info")
try:
mock_test_config[tb_key][mock_ctrlr_config_name] = [
"magic1", "magic2"
]
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
tr.register_controller(mock_controller)
self.assertEqual(tr.results.controller_info, {})
finally:
setattr(mock_controller, "get_info", get_info)
def test_controller_object_not_persistent_across_classes(self):
test_run_config = self.base_mock_test_config.copy()
test_run_config.controller_configs = {'MagicDevice': [{'serial': 1}]}
class FooTest(base_test.BaseTestClass):
def setup_class(cls1):
self.controller1 = cls1.register_controller(mock_controller)[0]
class BarTest(base_test.BaseTestClass):
def setup_class(cls2):
self.controller2 = cls2.register_controller(mock_controller)[0]
tr = test_runner.TestRunner(self.tmp_dir, test_run_config.testbed_name)
with tr.mobly_logger():
tr.add_test_class(test_run_config, FooTest)
tr.add_test_class(test_run_config, BarTest)
tr.run()
self.assertIsNot(self.controller1, self.controller2)
def test_run_twice_for_two_sets_of_logs(self, mock_timestamp):
"""Verifies the expected output files from a test run.
* Files are correctly created.
* Basic sanity checks of each output file.
"""
mock_test_config = self.create_mock_test_config(
self.base_mock_test_config)
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.setup_logger()
tr.run()
output_dir1 = logging.log_path
tr.run()
output_dir2 = logging.log_path
self.assertNotEqual(output_dir1, output_dir2)
self.assert_output_logs_exist(output_dir1)
self.assert_output_logs_exist(output_dir2)
def run_suite(test_classes, argv=None):
"""Executes multiple test classes as a suite.
This is the default entry point for running a test suite script file
directly.
Args:
test_classes: List of python classes containing Mobly tests.
argv: A list that is then parsed as cli args. If None, defaults to cli
input.
"""
args = _parse_cli_args(argv)
# Load test config file.
test_configs = config_parser.load_test_config_file(args.config)
# Check the classes that were passed in
for test_class in test_classes:
if not issubclass(test_class, base_test.BaseTestClass):
logging.error(
'Test class %s does not extend '
'mobly.base_test.BaseTestClass', test_class)
sys.exit(1)
# Find the full list of tests to execute
selected_tests = compute_selected_tests(test_classes, args.tests)
# Execute the suite
ok = True
for config in test_configs:
runner = test_runner.TestRunner(config.log_path, config.testbed_name)
with runner.mobly_logger():
for (test_class, tests) in selected_tests.items():
runner.add_test_class(config, test_class, tests)
try:
runner.run()
ok = runner.results.is_all_pass and ok
except signals.TestAbortAll:
pass
except Exception:
logging.exception('Exception when executing %s.',
config.testbed_name)
ok = False
if not ok:
sys.exit(1)
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(mock_test_config,
[('IntegrationTest', None)])
tr.run([integration_test.IntegrationTest])
self.assertFalse(tr.controller_registry)
self.assertFalse(tr.controller_destructors)
self.assertTrue(
mock_test_config.controller_configs[mock_ctrlr_config_name][0])
tr.run([integration_test.IntegrationTest])
tr.stop()
self.assertFalse(tr.controller_registry)
self.assertFalse(tr.controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
expected_info = {
'MagicDevice': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}]
}
self.assertEqual(tr.results.controller_info, expected_info)
def test_shortcut(self):
"""Verifies the shortcut is created and links properly."""
shortcut_path = os.path.join(self.log_dir, self.testbed_name,
'latest.lnk')
shell = client.Dispatch("WScript.Shell")
shortcut = shell.CreateShortCut(shortcut_path)
self.assertFalse(shortcut.Targetpath)
mock_test_config = self.create_mock_test_config(
self.base_mock_test_config)
tr = test_runner.TestRunner(self.log_dir, self.testbed_name)
with tr.mobly_logger():
pass
shortcut = shell.CreateShortCut(shortcut_path)
# Normalize paths for case and truncation
normalized_shortcut_path = os.path.normcase(
win32file.GetLongPathName(shortcut.Targetpath))
normalized_logger_path = os.path.normcase(
win32file.GetLongPathName(logging.log_path))
self.assertEqual(normalized_shortcut_path, normalized_logger_path)
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = dict(self.base_mock_test_config)
tb_key = keys.Config.key_testbed.value
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
"serial": "xxxx",
"magic": "Magic1"
}, {
"serial": "xxxx",
"magic": "Magic2"
}]
mock_test_config[tb_key][mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(mock_test_config,
[('IntegrationTest', None)])
tr.run()
self.assertFalse(tr.controller_registry)
self.assertFalse(tr.controller_destructors)
self.assertTrue(mock_test_config[tb_key][mock_ctrlr_config_name][0])
tr.run()
tr.stop()
self.assertFalse(tr.controller_registry)
self.assertFalse(tr.controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results["Requested"], 2)
self.assertEqual(results["Executed"], 2)
self.assertEqual(results["Passed"], 2)
expected_info = {
'MagicDevice': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}]
}
self.assertEqual(tr.results.controller_info, expected_info)
