def __init__(self, tmp_dir, user_params={}):
mock_test_run_configs = config_parser.TestRunConfig()
mock_test_run_configs.summary_writer = mock.Mock()
mock_test_run_configs.log_path = tmp_dir
mock_test_run_configs.user_params = user_params
mock_test_run_configs.reporter = mock.MagicMock()
super().__init__(mock_test_run_configs)
def setUp(self):
self.mock_test_cls_configs = config_parser.TestRunConfig()
self.mock_test_cls_configs.summary_writer = mock.Mock()
self.mock_test_cls_configs.log_path = '/tmp'
self.mock_test_cls_configs.user_params = {"some_param": "hahaha"}
self.mock_test_cls_configs.reporter = mock.MagicMock()
self.mock_test_name = "test_something"
def test_e2e(self, *args):
# Configure the test
test_to_mock = 'test_lte_traffic_direction_dlul_blimit_0_0'
self.tb_key = 'testbed_configs'
test_run_config = mobly_config_parser.TestRunConfig()
test_run_config.testbed_name = 'MockTestBed'
test_run_config.log_path = '/tmp'
test_run_config.summary_writer = mock.MagicMock()
test = self.PTTT(test_run_config)
mock_android = mock.Mock()
mock_android.model = 'coral'
test.unpack_userparams(
android_devices=[mock_android],
monsoons=[mock.Mock()],
iperf_servers=[mock.Mock(), mock.Mock()],
packet_senders=[mock.Mock(), mock.Mock()],
custom_files=[
'pass_fail_threshold_coral.json', 'rockbottom_coral.sh'
],
simulation=mock.Mock(spec=LteSimulation),
mon_freq=5000,
mon_duration=0,
mon_offset=0,
current_test_name=test_to_mock,
test_name=test_to_mock,
test_result=mock.Mock(),
bug_report={},
dut_rockbottom=mock.Mock(),
start_tel_traffic=mock.Mock(),
init_simulation=mock.Mock(),
initialize_simulator=mock.Mock(return_value=mock.Mock(
spec=cmw.CMW500CellularSimulator)),
collect_power_data=mock.Mock(),
get_iperf_results=mock.Mock(return_value={
'ul': 0,
'dl': 0
}),
pass_fail_check=mock.Mock())
# Emulate lifecycle
test.setup_class()
test.setup_test()
test.power_tel_traffic_test()
test.teardown_test()
test.teardown_class()
self.assertTrue(test.start_tel_traffic.called,
'Start traffic was not called')
self.assertTrue(test.init_simulation.called,
'Simulation was not initialized')
self.assertTrue(test.initialize_simulator.called,
'Simulator was not initialized')
self.assertTrue(test.collect_power_data.called,
'Power data was not collected')
self.assertTrue(test.get_iperf_results.called,
'Did not get iperf results')
self.assertTrue(test.pass_fail_check.called,
'Pass/Fail check was not performed')
def setUp(self):
self.tb_key = 'testbed_configs'
test_run_config = mobly_config_parser.TestRunConfig()
test_run_config.testbed_name = 'MockTestBed'
test_run_config.log_path = '/tmp'
test_run_config.summary_writer = mock.MagicMock()
test = self.PCBT(test_run_config)
self.test = test
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.mock_test_cls_configs = config_parser.TestRunConfig()
self.summary_file = os.path.join(self.tmp_dir, 'summary.yaml')
self.mock_test_cls_configs.summary_writer = records.TestSummaryWriter(
self.summary_file)
self.mock_test_cls_configs.log_path = self.tmp_dir
self.mock_test_cls_configs.user_params = {"some_param": "hahaha"}
self.mock_test_cls_configs.reporter = mock.MagicMock()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
def setUpClass(cls):
cls.tmp_dir = tempfile.mkdtemp()
cls.MOCK_CONFIG = mobly_config_parser.TestRunConfig()
cls.MOCK_CONFIG.testbed_name = 'SampleTestBed'
cls.MOCK_CONFIG.log_path = cls.tmp_dir
cls.MOCK_TEST_RUN_LIST = [(MockTest.__name__,
[MockTest.TEST_CASE_LIST])]
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
self.mock_run_list = [('SampleTest', None)]
def test_test_class_subscribed_fn_receives_event(self):
"""Tests that TestClasses have their subscribed functions called."""
with tempfile.TemporaryDirectory() as tmp_dir:
test_run_config = mobly_config_parser.TestRunConfig()
test_run_config.testbed_name = 'SampleTestBed'
test_run_config.log_path = tmp_dir
TestRunner(test_run_config, [('TestClass', [])]).run(TestClass)
self.assertGreaterEqual(len(TestClass.instance_event_received), 1)
self.assertEqual(len(TestClass.static_event_received), 0)
def test_subscribe_instance_bundles(self):
"""Tests that @subscribe bundles register only instance listeners."""
test_run_config = mobly_config_parser.TestRunConfig()
test_run_config.testbed_name = ''
test_run_config.log_path = ''
test_object = TestClass(test_run_config)
bundle = subscription_bundle.create_from_instance(test_object)
bundle.register()
event_bus.post(Event())
self.assertEqual(len(TestClass.instance_event_received), 1)
self.assertEqual(len(TestClass.static_event_received), 0)
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.tb_key = 'testbed_configs'
self.test_run_config = mobly_config_parser.TestRunConfig()
self.test_run_config.testbed_name = 'SampleTestBed'
self.test_run_config.controller_configs = {
self.tb_key: {
'name': self.test_run_config.testbed_name,
},
}
self.test_run_config.log_path = self.tmp_dir
self.test_run_config.user_params = {'some_param': 'hahaha'}
self.test_run_config.summary_writer = mock.MagicMock()
self.mock_test_name = 'test_something'
def _print_test_names(test_class):
"""Prints the names of all the tests in a test module.
If the module has generated tests defined based on controller info, this
may not be able to print the generated tests.
Args:
test_class: module, the test module to print names from.
"""
cls = test_class(config_parser.TestRunConfig())
try:
cls.setup_generated_tests()
except:
logging.exception('Failed to retrieve generated tests.')
print('==========> %s <==========' % cls.TAG)
for name in cls.get_existing_test_names():
print(name)
def test_mobly_test(self, mock_set, mock_create_device):
"""Use a fake mobly test to verify everything actually works as expected."""
class FakeTest(base_test.BaseTestClass):
"""Fake Mobly test to verify functionality."""
def setup_class(self):
self.devices = self.register_controller(gazoo_device)
def test_1(self):
asserts.assert_true(len(self.devices) == 2, self.devices)
fake_config = config_parser.TestRunConfig()
fake_config.log_path = self.artifacts_directory
summary_file = os.path.join(self.artifacts_directory, "summary.yaml")
fake_config.summary_writer = records.TestSummaryWriter(summary_file)
fake_config.controller_configs = FAKE_CONTROLLER_CONFIGS
test = FakeTest(fake_config)
test.run(["test_1"])
self.assertTrue(test.results.passed,
"Test results: {}".format(test.results))
actual_record = test.results.passed[0]
self.assertIn(actual_record.test_name, "test_1")
class TestRunnerTest(unittest.TestCase):
"""This test class has unit tests for the implementation of everything
under mobly.test_runner.
"""
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
self.log_dir = self.base_mock_test_config.log_path
self.test_bed_name = self.base_mock_test_config.test_bed_name
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def _assertControllerInfoEqual(self, info, expected_info_dict):
self.assertEqual(expected_info_dict['Controller Name'],
info.controller_name)
self.assertEqual(expected_info_dict['Test Class'], info.test_class)
self.assertEqual(expected_info_dict['Controller Info'],
info.controller_info)
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
self.assertTrue(
mock_test_config.controller_configs[mock_ctrlr_config_name][0])
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
expected_info_dict = {
'Controller Info': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}],
'Controller Name':
'MagicDevice',
'Test Class':
'IntegrationTest',
}
self._assertControllerInfoEqual(tr.results.controller_info[0],
expected_info_dict)
self._assertControllerInfoEqual(tr.results.controller_info[1],
expected_info_dict)
self.assertNotEqual(tr.results.controller_info[0],
tr.results.controller_info[1])
def test_summary_file_entries(self):
"""Verifies the output summary's file format.
This focuses on the format of the file instead of the content of
entries, which is covered in base_test_test.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
summary_path = os.path.join(logging.log_path,
records.OUTPUT_FILE_SUMMARY)
with io.open(summary_path, 'r', encoding='utf-8') as f:
summary_entries = list(yaml.safe_load_all(f))
self.assertEqual(len(summary_entries), 4)
# Verify the first entry is the list of test names.
self.assertEqual(summary_entries[0]['Type'],
records.TestSummaryEntryType.TEST_NAME_LIST.value)
self.assertEqual(summary_entries[1]['Type'],
records.TestSummaryEntryType.RECORD.value)
self.assertEqual(summary_entries[2]['Type'],
records.TestSummaryEntryType.CONTROLLER_INFO.value)
self.assertEqual(summary_entries[3]['Type'],
records.TestSummaryEntryType.SUMMARY.value)
@mock.patch('mobly.controllers.android_device_lib.adb.AdbProxy',
return_value=mock_android_device.MockAdbProxy(1))
@mock.patch('mobly.controllers.android_device_lib.fastboot.FastbootProxy',
return_value=mock_android_device.MockFastbootProxy(1))
@mock.patch('mobly.controllers.android_device.list_adb_devices',
return_value=['1'])
@mock.patch('mobly.controllers.android_device.get_all_instances',
return_value=mock_android_device.get_mock_ads(1))
def test_run_two_test_classes(self, mock_get_all, mock_list_adb,
mock_fastboot, mock_adb):
"""Verifies that running more than one test class in one test run works
properly.
This requires using a built-in controller module. Using AndroidDevice
module since it has all the mocks needed already.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
mock_test_config.controller_configs['AndroidDevice'] = [{
'serial': '1'
}]
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration2_test.Integration2Test)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
# Tag of the test class defaults to the class name.
record1 = tr.results.executed[0]
record2 = tr.results.executed[1]
self.assertEqual(record1.test_class, 'Integration2Test')
self.assertEqual(record2.test_class, 'IntegrationTest')
def test_run_two_test_classes_different_configs_and_aliases(self):
"""Verifies that running more than one test class in one test run with
different configs works properly.
"""
config1 = self.base_mock_test_config.copy()
config1.controller_configs[
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME] = [{
'serial': 'xxxx'
}]
config2 = config1.copy()
config2.user_params['icecream'] = 10
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(config1,
integration_test.IntegrationTest,
name_suffix='FirstConfig')
tr.add_test_class(config2,
integration_test.IntegrationTest,
name_suffix='SecondConfig')
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 1)
self.assertEqual(results['Failed'], 1)
self.assertEqual(tr.results.failed[0].details, '10 != 42')
record1 = tr.results.executed[0]
record2 = tr.results.executed[1]
self.assertEqual(record1.test_class, 'IntegrationTest_FirstConfig')
self.assertEqual(record2.test_class, 'IntegrationTest_SecondConfig')
def test_run_with_abort_all(self):
mock_test_config = self.base_mock_test_config.copy()
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration3_test.Integration3Test)
with self.assertRaises(signals.TestAbortAll):
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 1)
self.assertEqual(results['Executed'], 0)
self.assertEqual(results['Passed'], 0)
self.assertEqual(results['Failed'], 0)
def test_add_test_class_mismatched_log_path(self):
tr = test_runner.TestRunner('/different/log/dir', self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s log folder is "/different/log/dir", but a test '
r'config with a different log folder \("%s"\) was added.' %
re.escape(self.log_dir)):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_add_test_class_mismatched_test_bed_name(self):
tr = test_runner.TestRunner(self.log_dir, 'different_test_bed')
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s test bed is "different_test_bed", but a test '
r'config with a different test bed \("%s"\) was added.' %
self.test_bed_name):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_teardown_logger_before_setup_logger(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
r'TestRunner\._teardown_logger\(\) called before '
r'TestRunner\.setup_logger\(\)!'):
tr._teardown_logger()
def test_run_no_tests(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(test_runner.Error, 'No tests to execute.'):
tr.run()
@mock.patch('mobly.test_runner._find_test_class',
return_value=type('SampleTest', (), {}))
@mock.patch('mobly.test_runner.config_parser.load_test_config_file',
return_value=[config_parser.TestRunConfig()])
@mock.patch('mobly.test_runner.TestRunner', return_value=mock.MagicMock())
def test_main_parse_args(self, mock_test_runner, mock_config,
mock_find_test):
test_runner.main(['-c', 'some/path/foo.yaml', '-b', 'hello'])
mock_config.assert_called_with('some/path/foo.yaml', None)
def test_run_config_type(self):
config = config_parser.TestRunConfig()
self.assertNotIn('summary_writer', str(config))
self.assertNotIn('register_controller', str(config))
def main():
test_run_config = mobly_config_parser.TestRunConfig()
test_run_config.testbed_name = 'UnitTests'
test_run_config.log_path = ''
with mock.patch('mobly.utils.create_dir'):
ActsUnitTest(test_run_config).test_units()
class TestRunnerTest(unittest.TestCase):
"""This test class has unit tests for the implementation of everything
under mobly.test_runner.
"""
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
self.log_dir = self.base_mock_test_config.log_path
self.test_bed_name = self.base_mock_test_config.test_bed_name
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def test_register_controller_no_config(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(signals.ControllerError,
'No corresponding config found for'):
tr._register_controller(self.base_mock_test_config,
mock_controller)
def test_register_controller_no_config_no_register(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
self.assertIsNone(
tr._register_controller(self.base_mock_test_config,
mock_controller,
required=False))
def test_register_controller_dup_register(self):
"""Verifies correctness of registration, internal tally of controllers
objects, and the right error happen when a controller module is
registered twice.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr._register_controller(mock_test_config, mock_controller)
registered_name = 'mock_controller'
self.assertTrue(registered_name in tr._controller_registry)
mock_ctrlrs = tr._controller_registry[registered_name]
self.assertEqual(mock_ctrlrs[0].magic, 'magic1')
self.assertEqual(mock_ctrlrs[1].magic, 'magic2')
self.assertTrue(tr._controller_destructors[registered_name])
expected_msg = 'Controller module .* has already been registered.'
with self.assertRaisesRegex(signals.ControllerError, expected_msg):
tr._register_controller(mock_test_config, mock_controller)
def test_register_controller_no_get_info(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
get_info = getattr(mock_controller, 'get_info')
delattr(mock_controller, 'get_info')
try:
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr._register_controller(mock_test_config, mock_controller)
self.assertEqual(tr.results.controller_info, {})
finally:
setattr(mock_controller, 'get_info', get_info)
def test_register_controller_return_value(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
magic_devices = tr._register_controller(mock_test_config,
mock_controller)
self.assertEqual(magic_devices[0].magic, 'magic1')
self.assertEqual(magic_devices[1].magic, 'magic2')
def test_register_controller_change_return_value(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
magic_devices = tr._register_controller(mock_test_config,
mock_controller)
magic1 = magic_devices.pop(0)
self.assertIs(magic1, tr._controller_registry['mock_controller'][0])
self.assertEqual(len(tr._controller_registry['mock_controller']), 2)
def test_register_controller_less_than_min_number(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
expected_msg = 'Expected to get at least 3 controller objects, got 2.'
with self.assertRaisesRegex(signals.ControllerError, expected_msg):
tr._register_controller(mock_test_config,
mock_controller,
min_number=3)
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
self.assertFalse(tr._controller_registry)
self.assertFalse(tr._controller_destructors)
self.assertTrue(
mock_test_config.controller_configs[mock_ctrlr_config_name][0])
tr.run()
self.assertFalse(tr._controller_registry)
self.assertFalse(tr._controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
expected_info = {
'MagicDevice': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}]
}
self.assertEqual(tr.results.controller_info, expected_info)
def test_summary_file_entries(self):
"""Verifies the output summary's file format.
This focuses on the format of the file instead of the content of
entries, which is covered in base_test_test.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
summary_path = os.path.join(mock_test_config.log_path,
mock_test_config.test_bed_name, 'latest',
records.OUTPUT_FILE_SUMMARY)
with open(summary_path, 'r') as f:
summary_entries = list(yaml.load_all(f))
self.assertEqual(len(summary_entries), 4)
# Verify the first entry is the list of test names.
self.assertEqual(summary_entries[0]['Type'],
records.TestSummaryEntryType.TEST_NAME_LIST.value)
self.assertEqual(summary_entries[1]['Type'],
records.TestSummaryEntryType.RECORD.value)
@mock.patch('mobly.controllers.android_device_lib.adb.AdbProxy',
return_value=mock_android_device.MockAdbProxy(1))
@mock.patch('mobly.controllers.android_device_lib.fastboot.FastbootProxy',
return_value=mock_android_device.MockFastbootProxy(1))
@mock.patch('mobly.controllers.android_device.list_adb_devices',
return_value=['1'])
@mock.patch('mobly.controllers.android_device.get_all_instances',
return_value=mock_android_device.get_mock_ads(1))
def test_run_two_test_classes(self, mock_get_all, mock_list_adb,
mock_fastboot, mock_adb):
"""Verifies that running more than one test class in one test run works
properly.
This requires using a built-in controller module. Using AndroidDevice
module since it has all the mocks needed already.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
mock_test_config.controller_configs['AndroidDevice'] = [{
'serial': '1'
}]
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration2_test.Integration2Test)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
self.assertFalse(tr._controller_registry)
self.assertFalse(tr._controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
def test_run_two_test_classes_different_configs(self):
"""Verifies that running more than one test class in one test run with
different configs works properly.
"""
config1 = self.base_mock_test_config.copy()
config1.controller_configs[
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME] = [{
'serial': 'xxxx'
}]
config2 = config1.copy()
config2.user_params['icecream'] = 10
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(config1, integration_test.IntegrationTest)
tr.add_test_class(config2, integration_test.IntegrationTest)
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 1)
self.assertEqual(results['Failed'], 1)
self.assertEqual(tr.results.failed[0].details, '10 != 42')
def test_run_with_abort_all(self):
mock_test_config = self.base_mock_test_config.copy()
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration3_test.Integration3Test)
with self.assertRaises(signals.TestAbortAll):
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 1)
self.assertEqual(results['Executed'], 0)
self.assertEqual(results['Passed'], 0)
self.assertEqual(results['Failed'], 0)
def test_add_test_class_mismatched_log_path(self):
tr = test_runner.TestRunner('/different/log/dir', self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s log folder is "/different/log/dir", but a test '
r'config with a different log folder \("%s"\) was added.' %
self.log_dir):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_add_test_class_mismatched_test_bed_name(self):
tr = test_runner.TestRunner(self.log_dir, 'different_test_bed')
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s test bed is "different_test_bed", but a test '
r'config with a different test bed \("%s"\) was added.' %
self.test_bed_name):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_teardown_logger_before_setup_logger(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\._teardown_logger\(\) called before '
'TestRunner\.setup_logger\(\)!'):
tr._teardown_logger()
def test_run_no_tests(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(test_runner.Error, 'No tests to execute.'):
tr.run()
def test_verify_controller_module(self):
test_runner.verify_controller_module(mock_controller)
def test_verify_controller_module_null_attr(self):
try:
tmp = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME = None
msg = 'Controller interface .* in .* cannot be null.'
with self.assertRaisesRegex(signals.ControllerError, msg):
test_runner.verify_controller_module(mock_controller)
finally:
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME = tmp
def test_verify_controller_module_missing_attr(self):
try:
tmp = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
delattr(mock_controller, 'MOBLY_CONTROLLER_CONFIG_NAME')
msg = 'Module .* missing required controller module attribute'
with self.assertRaisesRegex(signals.ControllerError, msg):
test_runner.verify_controller_module(mock_controller)
finally:
setattr(mock_controller, 'MOBLY_CONTROLLER_CONFIG_NAME', tmp)
@mock.patch('mobly.test_runner._find_test_class',
return_value=type('SampleTest', (), {}))
@mock.patch('mobly.test_runner.config_parser.load_test_config_file',
return_value=[config_parser.TestRunConfig()])
@mock.patch('mobly.test_runner.TestRunner', return_value=mock.MagicMock())
def test_main_parse_args(self, mock_test_runner, mock_config,
mock_find_test):
test_runner.main(['-c', 'some/path/foo.yaml', '-b', 'hello'])
mock_config.assert_called_with('some/path/foo.yaml', None)
def test_run_config_user_params_is_already_initialized(self):
config = config_parser.TestRunConfig()
expected_value = 'SOME_VALUE'
self.assertEqual(
config.user_params.get('NON_EXISTENT_KEY', expected_value),
expected_value)
