def test_start_services_on_ads(self):
"""Makes sure when an AndroidDevice fails to start some services, all
AndroidDevice objects get cleaned up.
"""
msg = "Some error happened."
ads = mock_android_device.get_mock_ads(3)
ads[0].start_services = mock.MagicMock()
ads[0].stop_services = mock.MagicMock()
ads[1].start_services = mock.MagicMock()
ads[1].stop_services = mock.MagicMock()
ads[2].start_services = mock.MagicMock(
side_effect=android_device.Error(msg))
ads[2].stop_services = mock.MagicMock()
with self.assertRaisesRegex(android_device.Error, msg):
android_device._start_services_on_ads(ads)
ads[0].stop_services.assert_called_once_with()
ads[1].stop_services.assert_called_once_with()
ads[2].stop_services.assert_called_once_with()
class TestRunnerTest(unittest.TestCase):
"""This test class has unit tests for the implementation of everything
under mobly.test_runner.
"""
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
self.log_dir = self.base_mock_test_config.log_path
self.test_bed_name = self.base_mock_test_config.test_bed_name
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def test_register_controller_no_config(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(signals.ControllerError,
'No corresponding config found for'):
tr._register_controller(self.base_mock_test_config,
mock_controller)
def test_register_controller_no_config_no_register(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
self.assertIsNone(
tr._register_controller(self.base_mock_test_config,
mock_controller,
required=False))
def test_register_controller_dup_register(self):
"""Verifies correctness of registration, internal tally of controllers
objects, and the right error happen when a controller module is
registered twice.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr._register_controller(mock_test_config, mock_controller)
registered_name = 'mock_controller'
self.assertTrue(registered_name in tr._controller_registry)
mock_ctrlrs = tr._controller_registry[registered_name]
self.assertEqual(mock_ctrlrs[0].magic, 'magic1')
self.assertEqual(mock_ctrlrs[1].magic, 'magic2')
self.assertTrue(tr._controller_destructors[registered_name])
expected_msg = 'Controller module .* has already been registered.'
with self.assertRaisesRegex(signals.ControllerError, expected_msg):
tr._register_controller(mock_test_config, mock_controller)
def test_register_controller_no_get_info(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
get_info = getattr(mock_controller, 'get_info')
delattr(mock_controller, 'get_info')
try:
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr._register_controller(mock_test_config, mock_controller)
self.assertEqual(tr.results.controller_info, {})
finally:
setattr(mock_controller, 'get_info', get_info)
def test_register_controller_return_value(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
magic_devices = tr._register_controller(mock_test_config,
mock_controller)
self.assertEqual(magic_devices[0].magic, 'magic1')
self.assertEqual(magic_devices[1].magic, 'magic2')
def test_register_controller_change_return_value(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
magic_devices = tr._register_controller(mock_test_config,
mock_controller)
magic1 = magic_devices.pop(0)
self.assertIs(magic1, tr._controller_registry['mock_controller'][0])
self.assertEqual(len(tr._controller_registry['mock_controller']), 2)
def test_register_controller_less_than_min_number(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
expected_msg = 'Expected to get at least 3 controller objects, got 2.'
with self.assertRaisesRegex(signals.ControllerError, expected_msg):
tr._register_controller(mock_test_config,
mock_controller,
min_number=3)
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
self.assertFalse(tr._controller_registry)
self.assertFalse(tr._controller_destructors)
self.assertTrue(
mock_test_config.controller_configs[mock_ctrlr_config_name][0])
tr.run()
self.assertFalse(tr._controller_registry)
self.assertFalse(tr._controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
expected_info = {
'MagicDevice': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}]
}
self.assertEqual(tr.results.controller_info, expected_info)
def test_summary_file_entries(self):
"""Verifies the output summary's file format.
This focuses on the format of the file instead of the content of
entries, which is covered in base_test_test.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
summary_path = os.path.join(mock_test_config.log_path,
mock_test_config.test_bed_name, 'latest',
records.OUTPUT_FILE_SUMMARY)
with open(summary_path, 'r') as f:
summary_entries = list(yaml.load_all(f))
self.assertEqual(len(summary_entries), 4)
# Verify the first entry is the list of test names.
self.assertEqual(summary_entries[0]['Type'],
records.TestSummaryEntryType.TEST_NAME_LIST.value)
self.assertEqual(summary_entries[1]['Type'],
records.TestSummaryEntryType.RECORD.value)
@mock.patch('mobly.controllers.android_device_lib.adb.AdbProxy',
return_value=mock_android_device.MockAdbProxy(1))
@mock.patch('mobly.controllers.android_device_lib.fastboot.FastbootProxy',
return_value=mock_android_device.MockFastbootProxy(1))
@mock.patch('mobly.controllers.android_device.list_adb_devices',
return_value=['1'])
@mock.patch('mobly.controllers.android_device.get_all_instances',
return_value=mock_android_device.get_mock_ads(1))
def test_run_two_test_classes(self, mock_get_all, mock_list_adb,
mock_fastboot, mock_adb):
"""Verifies that running more than one test class in one test run works
properly.
This requires using a built-in controller module. Using AndroidDevice
module since it has all the mocks needed already.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
mock_test_config.controller_configs['AndroidDevice'] = [{
'serial': '1'
}]
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration2_test.Integration2Test)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
self.assertFalse(tr._controller_registry)
self.assertFalse(tr._controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
def test_run_two_test_classes_different_configs(self):
"""Verifies that running more than one test class in one test run with
different configs works properly.
"""
config1 = self.base_mock_test_config.copy()
config1.controller_configs[
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME] = [{
'serial': 'xxxx'
}]
config2 = config1.copy()
config2.user_params['icecream'] = 10
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(config1, integration_test.IntegrationTest)
tr.add_test_class(config2, integration_test.IntegrationTest)
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 1)
self.assertEqual(results['Failed'], 1)
self.assertEqual(tr.results.failed[0].details, '10 != 42')
def test_run_with_abort_all(self):
mock_test_config = self.base_mock_test_config.copy()
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration3_test.Integration3Test)
with self.assertRaises(signals.TestAbortAll):
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 1)
self.assertEqual(results['Executed'], 0)
self.assertEqual(results['Passed'], 0)
self.assertEqual(results['Failed'], 0)
def test_add_test_class_mismatched_log_path(self):
tr = test_runner.TestRunner('/different/log/dir', self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s log folder is "/different/log/dir", but a test '
r'config with a different log folder \("%s"\) was added.' %
self.log_dir):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_add_test_class_mismatched_test_bed_name(self):
tr = test_runner.TestRunner(self.log_dir, 'different_test_bed')
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s test bed is "different_test_bed", but a test '
r'config with a different test bed \("%s"\) was added.' %
self.test_bed_name):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_teardown_logger_before_setup_logger(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\._teardown_logger\(\) called before '
'TestRunner\.setup_logger\(\)!'):
tr._teardown_logger()
def test_run_no_tests(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(test_runner.Error, 'No tests to execute.'):
tr.run()
def test_verify_controller_module(self):
test_runner.verify_controller_module(mock_controller)
def test_verify_controller_module_null_attr(self):
try:
tmp = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME = None
msg = 'Controller interface .* in .* cannot be null.'
with self.assertRaisesRegex(signals.ControllerError, msg):
test_runner.verify_controller_module(mock_controller)
finally:
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME = tmp
def test_verify_controller_module_missing_attr(self):
try:
tmp = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
delattr(mock_controller, 'MOBLY_CONTROLLER_CONFIG_NAME')
msg = 'Module .* missing required controller module attribute'
with self.assertRaisesRegex(signals.ControllerError, msg):
test_runner.verify_controller_module(mock_controller)
finally:
setattr(mock_controller, 'MOBLY_CONTROLLER_CONFIG_NAME', tmp)
@mock.patch('mobly.test_runner._find_test_class',
return_value=type('SampleTest', (), {}))
@mock.patch('mobly.test_runner.config_parser.load_test_config_file',
return_value=[config_parser.TestRunConfig()])
@mock.patch('mobly.test_runner.TestRunner', return_value=mock.MagicMock())
def test_main_parse_args(self, mock_test_runner, mock_config,
mock_find_test):
test_runner.main(['-c', 'some/path/foo.yaml', '-b', 'hello'])
mock_config.assert_called_with('some/path/foo.yaml', None)
def test_create_with_pickup_all(self):
pick_all_token = android_device.ANDROID_DEVICE_PICK_ALL_TOKEN
actual_ads = android_device.create(pick_all_token)
for actual, expected in zip(actual_ads,
mock_android_device.get_mock_ads(5)):
self.assertEqual(actual.serial, expected.serial)
def test_get_device_too_many_matches(self):
ads = mock_android_device.get_mock_ads(5)
target_serial = ads[1].serial = ads[0].serial
expected_msg = r"More than one device matched: \['0', '0'\]"
with self.assertRaisesRegex(android_device.Error, expected_msg):
android_device.get_device(ads, serial=target_serial)
def test_get_device_no_match(self):
ads = mock_android_device.get_mock_ads(5)
expected_msg = ('Could not find a target device that matches condition'
": {'serial': 5}.")
with self.assertRaisesRegex(android_device.Error, expected_msg):
ad = android_device.get_device(ads, serial=len(ads))
def test_get_device_success_with_serial(self):
ads = mock_android_device.get_mock_ads(5)
expected_serial = '0'
ad = android_device.get_device(ads, serial=expected_serial)
self.assertEqual(ad.serial, expected_serial)
def test_get_devices_no_match(self):
ads = mock_android_device.get_mock_ads(5)
expected_msg = ('Could not find a target device that matches condition'
": {'label': 'selected'}.")
with self.assertRaisesRegex(android_device.Error, expected_msg):
selected_ads = android_device.get_devices(ads, label='selected')
class TestRunnerTest(unittest.TestCase):
"""This test class has unit tests for the implementation of everything
under mobly.test_runner.
"""
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
self.log_dir = self.base_mock_test_config.log_path
self.test_bed_name = self.base_mock_test_config.test_bed_name
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def _assertControllerInfoEqual(self, info, expected_info_dict):
self.assertEqual(expected_info_dict['Controller Name'],
info.controller_name)
self.assertEqual(expected_info_dict['Test Class'], info.test_class)
self.assertEqual(expected_info_dict['Controller Info'],
info.controller_info)
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
self.assertTrue(
mock_test_config.controller_configs[mock_ctrlr_config_name][0])
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
expected_info_dict = {
'Controller Info': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}],
'Controller Name':
'MagicDevice',
'Test Class':
'IntegrationTest',
}
self._assertControllerInfoEqual(tr.results.controller_info[0],
expected_info_dict)
self._assertControllerInfoEqual(tr.results.controller_info[1],
expected_info_dict)
self.assertNotEqual(tr.results.controller_info[0],
tr.results.controller_info[1])
def test_summary_file_entries(self):
"""Verifies the output summary's file format.
This focuses on the format of the file instead of the content of
entries, which is covered in base_test_test.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
summary_path = os.path.join(logging.log_path,
records.OUTPUT_FILE_SUMMARY)
with io.open(summary_path, 'r', encoding='utf-8') as f:
summary_entries = list(yaml.safe_load_all(f))
self.assertEqual(len(summary_entries), 4)
# Verify the first entry is the list of test names.
self.assertEqual(summary_entries[0]['Type'],
records.TestSummaryEntryType.TEST_NAME_LIST.value)
self.assertEqual(summary_entries[1]['Type'],
records.TestSummaryEntryType.RECORD.value)
self.assertEqual(summary_entries[2]['Type'],
records.TestSummaryEntryType.CONTROLLER_INFO.value)
self.assertEqual(summary_entries[3]['Type'],
records.TestSummaryEntryType.SUMMARY.value)
@mock.patch('mobly.controllers.android_device_lib.adb.AdbProxy',
return_value=mock_android_device.MockAdbProxy(1))
@mock.patch('mobly.controllers.android_device_lib.fastboot.FastbootProxy',
return_value=mock_android_device.MockFastbootProxy(1))
@mock.patch('mobly.controllers.android_device.list_adb_devices',
return_value=['1'])
@mock.patch('mobly.controllers.android_device.get_all_instances',
return_value=mock_android_device.get_mock_ads(1))
def test_run_two_test_classes(self, mock_get_all, mock_list_adb,
mock_fastboot, mock_adb):
"""Verifies that running more than one test class in one test run works
properly.
This requires using a built-in controller module. Using AndroidDevice
module since it has all the mocks needed already.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
mock_test_config.controller_configs['AndroidDevice'] = [{
'serial': '1'
}]
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration2_test.Integration2Test)
tr.add_test_class(mock_test_config, integration_test.IntegrationTest)
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
# Tag of the test class defaults to the class name.
record1 = tr.results.executed[0]
record2 = tr.results.executed[1]
self.assertEqual(record1.test_class, 'Integration2Test')
self.assertEqual(record2.test_class, 'IntegrationTest')
def test_run_two_test_classes_different_configs_and_aliases(self):
"""Verifies that running more than one test class in one test run with
different configs works properly.
"""
config1 = self.base_mock_test_config.copy()
config1.controller_configs[
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME] = [{
'serial': 'xxxx'
}]
config2 = config1.copy()
config2.user_params['icecream'] = 10
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(config1,
integration_test.IntegrationTest,
name_suffix='FirstConfig')
tr.add_test_class(config2,
integration_test.IntegrationTest,
name_suffix='SecondConfig')
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 1)
self.assertEqual(results['Failed'], 1)
self.assertEqual(tr.results.failed[0].details, '10 != 42')
record1 = tr.results.executed[0]
record2 = tr.results.executed[1]
self.assertEqual(record1.test_class, 'IntegrationTest_FirstConfig')
self.assertEqual(record2.test_class, 'IntegrationTest_SecondConfig')
def test_run_with_abort_all(self):
mock_test_config = self.base_mock_test_config.copy()
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
tr.add_test_class(mock_test_config, integration3_test.Integration3Test)
with self.assertRaises(signals.TestAbortAll):
tr.run()
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 1)
self.assertEqual(results['Executed'], 0)
self.assertEqual(results['Passed'], 0)
self.assertEqual(results['Failed'], 0)
def test_add_test_class_mismatched_log_path(self):
tr = test_runner.TestRunner('/different/log/dir', self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s log folder is "/different/log/dir", but a test '
r'config with a different log folder \("%s"\) was added.' %
re.escape(self.log_dir)):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_add_test_class_mismatched_test_bed_name(self):
tr = test_runner.TestRunner(self.log_dir, 'different_test_bed')
with self.assertRaisesRegex(
test_runner.Error,
'TestRunner\'s test bed is "different_test_bed", but a test '
r'config with a different test bed \("%s"\) was added.' %
self.test_bed_name):
tr.add_test_class(self.base_mock_test_config,
integration_test.IntegrationTest)
def test_teardown_logger_before_setup_logger(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(
test_runner.Error,
r'TestRunner\._teardown_logger\(\) called before '
r'TestRunner\.setup_logger\(\)!'):
tr._teardown_logger()
def test_run_no_tests(self):
tr = test_runner.TestRunner(self.log_dir, self.test_bed_name)
with self.assertRaisesRegex(test_runner.Error, 'No tests to execute.'):
tr.run()
@mock.patch('mobly.test_runner._find_test_class',
return_value=type('SampleTest', (), {}))
@mock.patch('mobly.test_runner.config_parser.load_test_config_file',
return_value=[config_parser.TestRunConfig()])
@mock.patch('mobly.test_runner.TestRunner', return_value=mock.MagicMock())
def test_main_parse_args(self, mock_test_runner, mock_config,
mock_find_test):
test_runner.main(['-c', 'some/path/foo.yaml', '-b', 'hello'])
mock_config.assert_called_with('some/path/foo.yaml', None)
class TestRunnerTest(unittest.TestCase):
"""This test class has unit tests for the implementation of everything
under mobly.test_runner.
"""
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.base_mock_test_config = config_parser.TestRunConfig()
self.base_mock_test_config.test_bed_name = 'SampleTestBed'
self.base_mock_test_config.controller_configs = {}
self.base_mock_test_config.user_params = {
'icecream': 42,
'extra_param': 'haha'
}
self.base_mock_test_config.log_path = self.tmp_dir
self.mock_run_list = [('SampleTest', None)]
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def test_register_controller_no_config(self):
tr = test_runner.TestRunner(self.base_mock_test_config,
self.mock_run_list)
with self.assertRaisesRegexp(signals.ControllerError,
'No corresponding config found for'):
tr.register_controller(mock_controller)
def test_register_controller_no_config_no_register(self):
tr = test_runner.TestRunner(self.base_mock_test_config,
self.mock_run_list)
self.assertIsNone(
tr.register_controller(mock_controller, required=False))
def test_register_controller_dup_register(self):
"""Verifies correctness of registration, internal tally of controllers
objects, and the right error happen when a controller module is
registered twice.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
tr.register_controller(mock_controller)
registered_name = 'mock_controller'
self.assertTrue(registered_name in tr.controller_registry)
mock_ctrlrs = tr.controller_registry[registered_name]
self.assertEqual(mock_ctrlrs[0].magic, 'magic1')
self.assertEqual(mock_ctrlrs[1].magic, 'magic2')
self.assertTrue(tr.controller_destructors[registered_name])
expected_msg = 'Controller module .* has already been registered.'
with self.assertRaisesRegexp(signals.ControllerError, expected_msg):
tr.register_controller(mock_controller)
def test_register_controller_no_get_info(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_ref_name = 'haha'
get_info = getattr(mock_controller, 'get_info')
delattr(mock_controller, 'get_info')
try:
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
tr.register_controller(mock_controller)
self.assertEqual(tr.results.controller_info, {})
finally:
setattr(mock_controller, 'get_info', get_info)
def test_register_controller_return_value(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
magic_devices = tr.register_controller(mock_controller)
self.assertEqual(magic_devices[0].magic, 'magic1')
self.assertEqual(magic_devices[1].magic, 'magic2')
def test_register_controller_less_than_min_number(self):
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_test_config.controller_configs = {
mock_ctrlr_config_name: ['magic1', 'magic2']
}
tr = test_runner.TestRunner(mock_test_config, self.mock_run_list)
expected_msg = 'Expected to get at least 3 controller objects, got 2.'
with self.assertRaisesRegexp(signals.ControllerError, expected_msg):
tr.register_controller(mock_controller, min_number=3)
def test_run_twice(self):
"""Verifies that:
1. Repeated run works properly.
2. The original configuration is not altered if a test controller
module modifies configuration.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
tr = test_runner.TestRunner(mock_test_config,
[('IntegrationTest', None)])
tr.run([integration_test.IntegrationTest])
self.assertFalse(tr.controller_registry)
self.assertFalse(tr.controller_destructors)
self.assertTrue(
mock_test_config.controller_configs[mock_ctrlr_config_name][0])
tr.run([integration_test.IntegrationTest])
tr.stop()
self.assertFalse(tr.controller_registry)
self.assertFalse(tr.controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
expected_info = {
'MagicDevice': [{
'MyMagic': {
'magic': 'Magic1'
}
}, {
'MyMagic': {
'magic': 'Magic2'
}
}]
}
self.assertEqual(tr.results.controller_info, expected_info)
@mock.patch('mobly.controllers.android_device_lib.adb.AdbProxy',
return_value=mock_android_device.MockAdbProxy(1))
@mock.patch('mobly.controllers.android_device_lib.fastboot.FastbootProxy',
return_value=mock_android_device.MockFastbootProxy(1))
@mock.patch('mobly.controllers.android_device.list_adb_devices',
return_value=['1'])
@mock.patch('mobly.controllers.android_device.get_all_instances',
return_value=mock_android_device.get_mock_ads(1))
def test_run_two_test_classes(
self,
mock_get_all,
mock_list_adb,
mock_fastboot,
mock_adb,
):
"""Verifies that runing more than one test class in one test run works
proerly.
This requires using a built-in controller module. Using AndroidDevice
module since it has all the mocks needed already.
"""
mock_test_config = self.base_mock_test_config.copy()
mock_ctrlr_config_name = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
my_config = [{
'serial': 'xxxx',
'magic': 'Magic1'
}, {
'serial': 'xxxx',
'magic': 'Magic2'
}]
mock_test_config.controller_configs[mock_ctrlr_config_name] = my_config
mock_test_config.controller_configs['AndroidDevice'] = [{
'serial': '1'
}]
tr = test_runner.TestRunner(mock_test_config,
[('Integration2Test', None),
('IntegrationTest', None)])
tr.run([
integration_test.IntegrationTest,
integration2_test.Integration2Test
])
tr.stop()
self.assertFalse(tr.controller_registry)
self.assertFalse(tr.controller_destructors)
results = tr.results.summary_dict()
self.assertEqual(results['Requested'], 2)
self.assertEqual(results['Executed'], 2)
self.assertEqual(results['Passed'], 2)
def test_verify_controller_module(self):
test_runner.TestRunner.verify_controller_module(mock_controller)
def test_verify_controller_module_null_attr(self):
try:
tmp = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME = None
msg = 'Controller interface .* in .* cannot be null.'
with self.assertRaisesRegexp(signals.ControllerError, msg):
test_runner.TestRunner.verify_controller_module(
mock_controller)
finally:
mock_controller.MOBLY_CONTROLLER_CONFIG_NAME = tmp
def test_verify_controller_module_missing_attr(self):
try:
tmp = mock_controller.MOBLY_CONTROLLER_CONFIG_NAME
delattr(mock_controller, 'MOBLY_CONTROLLER_CONFIG_NAME')
msg = 'Module .* missing required controller module attribute'
with self.assertRaisesRegexp(signals.ControllerError, msg):
test_runner.TestRunner.verify_controller_module(
mock_controller)
finally:
setattr(mock_controller, 'MOBLY_CONTROLLER_CONFIG_NAME', tmp)
