def test_is_all_pass_negative(self):
s = signals.TestFailure(self.details, self.float_extra)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(s)
record2 = records.TestResultRecord(self.tn)
record2.test_begin()
record2.test_error(s)
tr = records.TestResult()
tr.add_record(record1)
tr.add_record(record2)
self.assertFalse(tr.is_all_pass)
def test_result_fail_class_with_test_signal(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr = records.TestResult()
tr.add_record(record1)
s = signals.TestFailure(self.details, self.float_extra)
record2 = records.TestResultRecord("SomeTest", s)
tr.fail_class(record2)
self.assertEqual(len(tr.passed), 1)
self.assertEqual(len(tr.failed), 1)
self.assertEqual(len(tr.executed), 2)
def test_result_add_operator_success(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr1 = records.TestResult()
tr1.add_record(record1)
record2 = records.TestResultRecord(self.tn)
record2.test_begin()
s = signals.TestPass(self.details, self.json_extra)
record2.test_pass(s)
tr2 = records.TestResult()
tr2.add_record(record2)
tr2 += tr1
self.assertTrue(tr2.passed, [tr1, tr2])
def test_is_all_pass(self):
s = signals.TestPass(self.details, self.float_extra)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_pass(s)
s = signals.TestSkip(self.details, self.float_extra)
record2 = records.TestResultRecord(self.tn)
record2.test_begin()
record2.test_skip(s)
tr = records.TestResult()
tr.add_record(record1)
tr.add_record(record2)
tr.add_record(record1)
self.assertEqual(len(tr.passed), 2)
self.assertTrue(tr.is_all_pass)
def test_result_record_skip_none(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
record.test_skip()
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_SKIP,
details=None,
extras=None)
def test_result_record_pass_none(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
record.test_pass()
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_PASS,
details=None,
extras=None)
def test_result_record_fail_none(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
record.test_fail()
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=None,
extras=None)
def test_result_add_operator_success(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr1 = records.TestResult()
tr1.add_record(record1)
tr1.add_controller_info("MockDevice", ["magicA", "magicB"])
record2 = records.TestResultRecord(self.tn)
record2.test_begin()
s = signals.TestPass(self.details, self.json_extra)
record2.test_pass(s)
tr2 = records.TestResult()
tr2.add_record(record2)
tr2.add_controller_info("MockDevice", ["magicC"])
tr2 += tr1
self.assertTrue(tr2.passed, [tr1, tr2])
self.assertTrue(tr2.controller_info, {"MockDevice": ["magicC"]})
def test_result_record_fail_with_json_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestFailure(self.details, self.json_extra)
record.test_fail(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=self.details,
extras=self.json_extra)
def test_result_record_skip_with_json_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestSkip(self.details, self.json_extra)
record.test_skip(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_SKIP,
details=self.details,
extras=self.json_extra)
def test_result_record_pass_with_json_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestPass(self.details, self.json_extra)
record.test_pass(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_PASS,
details=self.details,
extras=self.json_extra)
def test_result_add_operator_type_mismatch(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr1 = records.TestResult()
tr1.add_record(record1)
expected_msg = "Operand .* of type .* is not a TestResult."
with self.assertRaisesRegexp(TypeError, expected_msg):
tr1 += "haha"
def test_is_all_pass_with_fail_class(self):
"""Verifies that is_all_pass yields correct value when fail_class is
used.
"""
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(Exception("haha"))
tr = records.TestResult()
tr.fail_class(record1)
self.assertFalse(tr.is_all_pass)
def test_result_add_operator_success(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr1 = records.TestResult()
tr1.add_record(record1)
device1 = ControllerInfoRecord('TestClass', 'MockDevice', 'device1')
tr1.add_controller_info_record(device1)
record2 = records.TestResultRecord(self.tn)
record2.test_begin()
s = signals.TestPass(self.details, self.json_extra)
record2.test_pass(s)
tr2 = records.TestResult()
tr2.add_record(record2)
device2 = ControllerInfoRecord('TestClass', 'MockDevice', 'device2')
tr2.add_controller_info_record(device2)
tr2 += tr1
self.assertTrue(tr2.passed, [tr1, tr2])
self.assertTrue(tr2.controller_info, [device1, device2])
def _block_all_test_cases(self, tests):
"""Over-write _block_all_test_case in BaseTestClass."""
for (i, (test_name, test_func)) in enumerate(tests):
signal = signals.TestBlocked("Failed class setup")
record = records.TestResultRecord(test_name, self.TAG)
record.test_begin()
# mark all test cases as FAIL
record.test_fail(signal)
self.results.add_record(record)
# only gather bug report for the first test case
if i == 0:
self.on_fail(test_name, record.begin_time)
def test_result_fail_class_with_special_error(self):
"""Call TestResult.fail_class with an error class that requires more
than one arg to instantiate.
"""
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr = records.TestResult()
tr.add_record(record1)
class SpecialError(Exception):
def __init__(self, arg1, arg2):
self.msg = "%s %s" % (arg1, arg2)
se = SpecialError("haha", 42)
record2 = records.TestResultRecord("SomeTest", se)
tr.fail_class(record2)
self.assertEqual(len(tr.passed), 1)
self.assertEqual(len(tr.failed), 1)
self.assertEqual(len(tr.executed), 2)
def _block_all_test_cases(self, tests):
"""
Block all passed in test cases.
Args:
tests: The tests to block.
"""
for test_name, test_func in tests:
signal = signals.TestBlocked("Failed class setup")
record = records.TestResultRecord(test_name, self.TAG)
record.test_begin()
if hasattr(test_func, 'gather'):
signal.extras = test_func.gather()
record.test_blocked(signal)
self.results.add_record(record)
self._on_blocked(record)
def run_test_class(self, test_cls_name, test_cases=None):
"""Instantiates and executes a test class.
If test_cases is None, the test cases listed by self.tests will be
executed instead. If self.tests is empty as well, no test case in this
test class will be executed.
Args:
test_cls_name: Name of the test class to execute.
test_cases: List of test case names to execute within the class.
Raises:
ValueError is raised if the requested test class could not be found
in the test_paths directories.
"""
matches = fnmatch.filter(self.test_classes.keys(), test_cls_name)
if not matches:
self.log.info(
"Cannot find test class %s or classes matching pattern, "
"skipping for now." % test_cls_name)
record = records.TestResultRecord("*all*", test_cls_name)
record.test_skip(signals.TestSkip("Test class does not exist."))
self.results.add_record(record)
return
if matches != [test_cls_name]:
self.log.info("Found classes matching pattern %s: %s",
test_cls_name, matches)
for test_cls_name_match in matches:
test_cls = self.test_classes[test_cls_name_match]
if self.test_configs.get(keys.Config.key_random.value) or (
"Preflight"
in test_cls_name_match) or ("Postflight"
in test_cls_name_match):
test_case_iterations = 1
else:
test_case_iterations = self.test_configs.get(
keys.Config.key_test_case_iterations.value, 1)
with test_cls(self.test_run_info) as test_cls_instance:
try:
cls_result = test_cls_instance.run(test_cases,
test_case_iterations)
self.results += cls_result
self._write_results_json_str()
except signals.TestAbortAll as e:
self.results += e.results
raise e
def _block_all_test_cases(self, tests, reason='Failed class setup'):
"""
Block all passed in test cases.
Args:
tests: The tests to block.
reason: Message describing the reason that the tests are blocked.
Default is 'Failed class setup'
"""
for test_name, test_func in tests:
signal = signals.TestError(reason)
record = records.TestResultRecord(test_name, self.TAG)
record.test_begin()
if hasattr(test_func, 'gather'):
signal.extras = test_func.gather()
record.test_error(signal)
self.results.add_record(record)
self.summary_writer.dump(record.to_dict(),
records.TestSummaryEntryType.RECORD)
self._on_skip(record)
def run_test_class(self, test_cls_name, test_cases=None):
"""Instantiates and executes a test class.
If test_cases is None, the test cases listed by self.tests will be
executed instead. If self.tests is empty as well, no test case in this
test class will be executed.
Args:
test_cls_name: Name of the test class to execute.
test_cases: List of test case names to execute within the class.
Raises:
ValueError is raised if the requested test class could not be found
in the test_paths directories.
"""
matches = fnmatch.filter(self.test_classes.keys(), test_cls_name)
if not matches:
self.log.info(
'Cannot find test class %s or classes matching pattern, '
'skipping for now.' % test_cls_name)
record = records.TestResultRecord('*all*', test_cls_name)
record.test_skip(signals.TestSkip('Test class does not exist.'))
self.results.add_record(record)
return
if matches != [test_cls_name]:
self.log.info('Found classes matching pattern %s: %s',
test_cls_name, matches)
for test_cls_name_match in matches:
test_cls = self.test_classes[test_cls_name_match]
test_cls_instance = test_cls(self.test_run_config)
try:
cls_result = test_cls_instance.run(test_cases)
self.results += cls_result
except signals.TestAbortAll as e:
self.results += e.results
raise e
def exec_one_testcase(self, test_name, test_func):
"""Executes one test case and update test results.
Executes one test case, create a records.TestResultRecord object with
the execution information, and add the record to the test class's test
results.
Args:
test_name: Name of the test.
test_func: The test function.
"""
class_name = self.__class__.__name__
tr_record = records.TestResultRecord(test_name, class_name)
tr_record.test_begin()
self.begin_time = int(tr_record.begin_time)
self.log_begin_time = tr_record.log_begin_time
self.test_name = tr_record.test_name
event_bus.post(TestCaseBeginEvent(self, self.test_name))
self.log.info("%s %s", TEST_CASE_TOKEN, test_name)
# Enable test retry if specified in the ACTS config
retry_tests = self.user_params.get('retry_tests', [])
full_test_name = '%s.%s' % (class_name, self.test_name)
if any(name in retry_tests for name in [class_name, full_test_name]):
test_func = self.get_func_with_retry(test_func)
verdict = None
test_signal = None
try:
try:
ret = self._setup_test(self.test_name)
asserts.assert_true(ret is not False,
"Setup for %s failed." % test_name)
verdict = test_func()
finally:
try:
self._teardown_test(self.test_name)
except signals.TestAbortAll:
raise
except Exception as e:
self.log.error(traceback.format_exc())
tr_record.add_error("teardown_test", e)
self._exec_procedure_func(self._on_exception, tr_record)
except (signals.TestFailure, AssertionError) as e:
test_signal = e
if self.user_params.get(
keys.Config.key_test_failure_tracebacks.value, False):
self.log.exception(e)
tr_record.test_fail(e)
self._exec_procedure_func(self._on_fail, tr_record)
except signals.TestSkip as e:
# Test skipped.
test_signal = e
tr_record.test_skip(e)
self._exec_procedure_func(self._on_skip, tr_record)
except (signals.TestAbortClass, signals.TestAbortAll) as e:
# Abort signals, pass along.
test_signal = e
tr_record.test_fail(e)
self._exec_procedure_func(self._on_fail, tr_record)
raise e
except signals.TestPass as e:
# Explicit test pass.
test_signal = e
tr_record.test_pass(e)
self._exec_procedure_func(self._on_pass, tr_record)
except Exception as e:
test_signal = e
self.log.error(traceback.format_exc())
# Exception happened during test.
tr_record.test_error(e)
self._exec_procedure_func(self._on_exception, tr_record)
self._exec_procedure_func(self._on_fail, tr_record)
else:
if verdict or (verdict is None):
# Test passed.
tr_record.test_pass()
self._exec_procedure_func(self._on_pass, tr_record)
return
tr_record.test_fail()
self._exec_procedure_func(self._on_fail, tr_record)
finally:
self.results.add_record(tr_record)
self.summary_writer.dump(tr_record.to_dict(),
records.TestSummaryEntryType.RECORD)
self.current_test_name = None
event_bus.post(TestCaseEndEvent(self, self.test_name, test_signal))
def run(self, test_names=None):
"""Runs test cases within a test class by the order they appear in the
execution list.
One of these test cases lists will be executed, shown here in priority
order:
1. The test_names list, which is passed from cmd line. Invalid names
are guarded by cmd line arg parsing.
2. The self.tests list defined in test class. Invalid names are
ignored.
3. All function that matches test case naming convention in the test
class.
Args:
test_names: A list of string that are test case names requested in
cmd line.
Returns:
The test results object of this class.
"""
self.log.info("==========> %s <==========", self.TAG)
# Devise the actual test cases to run in the test class.
if not test_names:
if self.tests:
# Specified by run list in class.
test_names = list(self.tests)
else:
# No test case specified by user, execute all in the test class
test_names = self._get_all_test_names()
self.results.requested = test_names
tests = self._get_test_funcs(test_names)
# A TestResultRecord used for when setup_class fails.
class_record = records.TestResultRecord("setup_class", self.TAG)
class_record.test_begin()
# Setup for the class.
try:
if self._setup_class() is False:
asserts.fail("Failed to setup %s." % self.TAG)
except Exception as e:
self.log.exception("Failed to setup %s.", self.TAG)
class_record.test_fail(e)
self._exec_procedure_func(self._on_fail, class_record)
self._exec_func(self.teardown_class)
self.results.fail_class(class_record)
return self.results
# Run tests in order.
try:
for test_name, test_func in tests:
self.exec_one_testcase(test_name, test_func, self.cli_args)
return self.results
except signals.TestAbortClass:
return self.results
except signals.TestAbortAll as e:
# Piggy-back test results on this exception object so we don't lose
# results from this test class.
setattr(e, "results", self.results)
raise e
finally:
self._exec_func(self.teardown_class)
self.log.info("Summary for test class %s: %s", self.TAG,
self.results.summary_str())
def exec_one_testcase(self, test_name, test_func, args, **kwargs):
"""Executes one test case and update test results.
Executes one test case, create a records.TestResultRecord object with
the execution information, and add the record to the test class's test
results.
Args:
test_name: Name of the test.
test_func: The test function.
args: A tuple of params.
kwargs: Extra kwargs.
"""
is_generate_trigger = False
tr_record = records.TestResultRecord(test_name, self.TAG)
tr_record.test_begin()
self.begin_time = tr_record.log_begin_time
self.test_name = tr_record.test_name
self.log.info("%s %s", TEST_CASE_TOKEN, test_name)
verdict = None
try:
try:
if hasattr(self, 'android_devices'):
for ad in self.android_devices:
if not ad.is_adb_logcat_on:
ad.start_adb_logcat(cont_logcat_file=True)
ret = self._setup_test(self.test_name)
asserts.assert_true(ret is not False,
"Setup for %s failed." % test_name)
if args or kwargs:
verdict = test_func(*args, **kwargs)
else:
verdict = test_func()
finally:
try:
self._teardown_test(self.test_name)
except signals.TestAbortAll:
raise
except Exception as e:
self.log.error(traceback.format_exc())
tr_record.add_error("teardown_test", e)
self._exec_procedure_func(self._on_exception, tr_record)
except (signals.TestFailure, AssertionError) as e:
self.log.error(e)
tr_record.test_fail(e)
self._exec_procedure_func(self._on_fail, tr_record)
except signals.TestSkip as e:
# Test skipped.
tr_record.test_skip(e)
self._exec_procedure_func(self._on_skip, tr_record)
except (signals.TestAbortClass, signals.TestAbortAll) as e:
# Abort signals, pass along.
tr_record.test_fail(e)
self._exec_procedure_func(self._on_fail, tr_record)
raise e
except signals.TestPass as e:
# Explicit test pass.
tr_record.test_pass(e)
self._exec_procedure_func(self._on_pass, tr_record)
except signals.TestSilent as e:
# This is a trigger test for generated tests, suppress reporting.
is_generate_trigger = True
self.results.requested.remove(test_name)
except signals.TestBlocked as e:
tr_record.test_blocked(e)
self._exec_procedure_func(self._on_blocked, tr_record)
except Exception as e:
self.log.error(traceback.format_exc())
# Exception happened during test.
tr_record.test_unknown(e)
self._exec_procedure_func(self._on_exception, tr_record)
self._exec_procedure_func(self._on_fail, tr_record)
else:
# Keep supporting return False for now.
# TODO(angli): Deprecate return False support.
if verdict or (verdict is None):
# Test passed.
tr_record.test_pass()
self._exec_procedure_func(self._on_pass, tr_record)
return
# Test failed because it didn't return True.
# This should be removed eventually.
tr_record.test_fail()
self._exec_procedure_func(self._on_fail, tr_record)
finally:
if not is_generate_trigger:
self.results.add_record(tr_record)
