def _RunSuite(self, test_exprs, suite_chroot_results_dir, timeout):
"""Runs a collection of tests.
Args:
test_exprs: List of string expressions describing which tests to run; this
is passed directly to the 'tast run' command. See
https://goo.gl/UPNEgT for info about test expressions.
suite_chroot_results_dir: String containing path of directory where the
tast command should store test results,
relative to chroot.
timeout: Integer containing timeout in seconds to pass to Tast. This is
used to let the Tast process reserve adequate time to collect
system information after running tests so that it can exit
cleanly instead of being killed.
Raises:
failures_lib.TestFailure if an internal error is encountered.
"""
vm_path = os.path.join(self.GetImageDirSymlink(), constants.VM_IMAGE_BIN)
results_dir = self._MakeChrootPathAbsolute(suite_chroot_results_dir)
cmd = ['./cros_run_test', '--no-display', '--copy-on-write', '--debug',
'--board=%s' % self._current_board, '--image-path=%s' % vm_path,
'--results-dir=%s' % results_dir, '--test-timeout=%d' % timeout,
'--tast'] + test_exprs
result = cros_build_lib.run(
cmd, error_code_ok=True, cwd=constants.CHROMITE_BIN_DIR,
kill_timeout=TastVMTestStage.CLEANUP_TIMEOUT_SEC
)
if result.returncode:
raise failures_lib.TestFailure(FAILURE_EXIT_CODE % result.returncode)
def testTastVMTestStageFails(self):
"""Verify VMTestStage is still run when TastVMTestStage fails."""
self.stage_exceptions = {
tast_test_stages.TastVMTestStage: failures_lib.TestFailure(),
}
self.assertEquals([failures_lib.TestFailure], self._RunVMTests())
self.assertEquals(self.all_vm_test_stages, self.called_stages)
def _RunTestSuiteUsingCtest(buildroot,
board,
image_path,
results_dir,
test_config,
allow_chrome_crashes,
ssh_private_key=None,
ssh_port=9228):
"""Runs the test harness suite using the ctest code path."""
results_dir_in_chroot = os.path.join(buildroot, 'chroot',
results_dir.lstrip('/'))
osutils.RmDir(results_dir_in_chroot, ignore_missing=True)
test_type = test_config.test_type
cwd = os.path.join(buildroot, 'src', 'scripts')
dut_type = 'gce' if test_type == constants.GCE_SUITE_TEST_TYPE else 'vm'
crostestutils = os.path.join(buildroot, 'src', 'platform', 'crostestutils')
cmd = [
os.path.join(crostestutils, 'au_test_harness',
'cros_au_test_harness.py'),
'--board=%s' % board,
'--type=%s' % dut_type, '--no_graphics', '--verbose',
'--target_image=%s' % image_path,
'--test_results_root=%s' % results_dir_in_chroot
]
if test_type not in constants.VALID_VM_TEST_TYPES:
raise AssertionError('Unrecognized test type %r' % test_type)
if test_type in [
constants.VM_SUITE_TEST_TYPE, constants.GCE_SUITE_TEST_TYPE
]:
cmd.append('--ssh_port=%s' % ssh_port)
cmd.append('--verify_suite_name=%s' % test_config.test_suite)
cmd.append('--test_prefix=SimpleTestVerify')
if allow_chrome_crashes:
cmd.append('--allow_chrome_crashes')
if ssh_private_key is not None:
cmd.append('--ssh_private_key=%s' % ssh_private_key)
# Give tests 10 minutes to clean up before shutting down.
result = cros_build_lib.run(cmd,
cwd=cwd,
check=False,
kill_timeout=10 * 60)
if result.returncode:
if os.path.exists(results_dir_in_chroot):
error = '%s exited with code %d' % (' '.join(cmd),
result.returncode)
with open(results_dir_in_chroot + '/failed_test_command',
'w') as failed:
failed.write(error)
raise failures_lib.TestFailure('** VMTests failed with code %d **' %
result.returncode)
def _ProcessResultsFile(self, abs_results_dir, url_base, suite_names):
"""Parses the results file and prints links to failed tests.
Args:
abs_results_dir: Absolute path to directory containing test results.
url_base: Relative path within the archive dir where results are stored.
suite_names: List of string test suite names.
Raises:
failures_lib.TestFailure if one or more tests failed or results were
missing or unreadable.
"""
num_failed = 0
for suite_name in sorted(suite_names):
results_path = os.path.join(abs_results_dir, suite_name,
RESULTS_FILENAME)
# The results file contains an array with objects representing tests.
# Each object should contain the test name in a 'name' attribute and a
# list of errors in an 'error' attribute.
try:
with open(results_path, 'r') as f:
for test in json.load(f):
# Report the test as failed if it didn't finish or had errors.
if test[RESULTS_END_KEY] == ZERO_TIME or test[
RESULTS_ERRORS_KEY]:
name = test[RESULTS_NAME_KEY]
informational = (RESULTS_INFORMATIONAL_ATTR
in test.get(RESULTS_ATTR_KEY, []))
test_url = os.path.join(url_base, suite_name,
RESULTS_TESTS_DIR, name)
desc = INFORMATIONAL_PREFIX + name if informational else name
self.PrintDownloadLink(test_url,
text_to_display=desc)
# Ignore the failure if the test was marked informational.
if not informational:
num_failed += 1
except Exception as e:
raise failures_lib.TestFailure(FAILURE_BAD_RESULTS %
(results_path, str(e)))
if num_failed > 0:
logging.error('%d test(s) failed', num_failed)
raise failures_lib.TestFailure(FAILURE_TESTS_FAILED % num_failed)
def ValidateMoblabVmTest(results_dir):
"""Determine if the VM test passed or not.
Args:
results_dir (str): Path to directory containing test_that results.
Raises:
failures_lib.TestFailure: If dummy_PassServer did not run or failed.
"""
log_file = os.path.join(results_dir, 'debug', 'test_that.INFO')
if not os.path.isfile(log_file):
raise failures_lib.TestFailure('Found no test_that logs at %s' % log_file)
log_file_contents = osutils.ReadFile(log_file)
if not re.match(r'dummy_PassServer\s*\[\s*PASSED\s*]', log_file_contents):
raise failures_lib.TestFailure('Moblab run_suite succeeded, but did '
'not successfully run dummy_PassServer.')
def testAllVMTestStagesFail(self):
"""Verify failures are reported when all VM test stages fail."""
self.stage_exceptions = {
vm_test_stages.VMTestStage: failures_lib.InfrastructureFailure(),
tast_test_stages.TastVMTestStage: failures_lib.TestFailure(),
}
self.assertEquals(
[failures_lib.InfrastructureFailure, failures_lib.TestFailure],
self._RunVMTests())
self.assertEquals(self.all_vm_test_stages, self.called_stages)
def ValidateMoblabTestSuccess(results_dir):
"""Verifies that moblab tests ran, and succeeded.
Looks at the result logs dropped by the moblab tests and sanity checks that
the expected tests ran, and were successful.
"""
log_path = os.path.join(results_dir, 'debug', 'test_that.INFO')
if not os.path.isfile(log_path):
raise failures_lib.TestFailure('Could not find test_that logs at %s' %
log_path)
dummy_pass_server_success_re = re.compile(
r'dummy_PassServer\s*\[\s*PASSED\s*]')
with open(log_path) as log_file:
for line in log_file:
if dummy_pass_server_success_re.search(line):
return
raise failures_lib.TestFailure(
'Moblab run_suite succeeded, but did not successfully run '
'dummy_PassServer')
def _RunTestSuiteUsingChromite(board,
image_path,
results_dir,
test_config,
allow_chrome_crashes,
ssh_private_key=None,
ssh_port=9228):
"""Runs the test harness suite using the chromite code path."""
image_dir = os.path.dirname(image_path)
vm_image_path = os.path.join(image_dir, constants.VM_IMAGE_BIN)
cmd = [
'cros_run_test',
'--debug',
'--board=%s' % board,
'--image-path=%s' % path_util.ToChrootPath(vm_image_path),
'--ssh-port=%s' % ssh_port,
'--autotest=suite:%s' % test_config.test_suite,
'--results-dir=%s' % results_dir,
]
if allow_chrome_crashes:
cmd.append('--test_that-args=--allow-chrome-crashes')
if ssh_private_key is not None:
cmd.append('--private-key=%s' %
path_util.ToChrootPath(ssh_private_key))
# Give tests 10 minutes to clean up before shutting down.
result = cros_build_lib.run(cmd,
check=False,
kill_timeout=10 * 60,
enter_chroot=True)
if result.returncode:
results_dir_in_chroot = os.path.join(constants.SOURCE_ROOT,
constants.DEFAULT_CHROOT_DIR,
results_dir.lstrip('/'))
if os.path.exists(results_dir_in_chroot):
error = '%s exited with code %d' % (' '.join(cmd),
result.returncode)
with open(results_dir_in_chroot + '/failed_test_command',
'w') as failed:
failed.write(error)
raise failures_lib.TestFailure('** VMTests failed with code %d **' %
result.returncode)
def _ProcessAndArchiveResults(self, abs_results_dir, suite_names,
already_have_error):
"""Processes and archives test results.
Args:
abs_results_dir: Absolute path to directory containing test results.
suite_names: List of string test suite names.
already_have_error: Boolean for whether testing has already failed.
Raises:
failures_lib.TestFailure if one or more tests failed or results were
unavailable. Suppressed if already_have_error is True.
"""
if not os.path.isdir(abs_results_dir) or not os.listdir(
abs_results_dir):
raise failures_lib.TestFailure(FAILURE_NO_RESULTS %
abs_results_dir)
archive_base = constants.TAST_VM_TEST_RESULTS % {
'attempt': self._attempt
}
_CopyResultsDir(abs_results_dir,
os.path.join(self.archive_path, archive_base))
# TODO(crbug.com/770562): Collect stack traces once the tast executable is
# symbolizing and collecting them (see VMTestStage._ArchiveTestResults).
# Now archive the results to Cloud Storage.
logging.info('Uploading artifacts to Cloud Storage...')
self.UploadArtifact(archive_base, archive=False, strict=False)
self.PrintDownloadLink(archive_base, RESULTS_LINK_PREFIX)
try:
self._ProcessResultsFile(abs_results_dir, archive_base,
suite_names)
except Exception as e:
# Don't raise a new exception if testing already failed.
if already_have_error:
logging.exception(
'Got error while archiving or processing results')
else:
raise e
finally:
osutils.RmDir(abs_results_dir, ignore_missing=True, sudo=True)
def _RunSuite(self, test_exprs, suite_chroot_results_dir):
"""Runs a collection of tests.
Args:
test_exprs: List of string expressions describing which tests to run; this
is passed directly to the 'tast run' command. See
https://goo.gl/UPNEgT for info about test expressions.
suite_chroot_results_dir: String containing path of directory where the
tast command should store test results,
relative to chroot.
Raises:
failures_lib.TestFailure if an internal error is encountered.
"""
image = os.path.join(self.GetImageDirSymlink(),
constants.TEST_IMAGE_BIN)
ssh_key = os.path.join(self.GetImageDirSymlink(),
constants.TEST_KEY_PRIVATE)
cmd = [
TastVMTestStage.SCRIPT_PATH,
'--board=' + self._current_board,
'--image_path=' + image,
'--ssh_private_key=' + ssh_key,
'--no_graphics',
'--results_dir=' + suite_chroot_results_dir,
]
cmd += test_exprs
result = cros_build_lib.RunCommand(
cmd,
cwd=os.path.join(self._build_root, 'src/scripts'),
error_code_ok=True,
kill_timeout=TastVMTestStage.CLEANUP_TIMEOUT_SEC)
if result.returncode:
raise failures_lib.TestFailure(FAILURE_EXIT_CODE %
result.returncode)
def _RunHWTestSuite(self,
debug=False,
fails=False,
warns=False,
cmd_fail_mode=None):
"""Verify the stage behavior in various circumstances.
Args:
debug: Whether the HWTest suite should be run in debug mode.
fails: Whether the stage should fail.
warns: Whether the stage should warn.
cmd_fail_mode: How commands.RunHWTestSuite() should fail.
If None, don't fail.
"""
# We choose to define these mocks in setUp() because they are
# useful for tests that do not call this method. However, this
# means we have to reset the mocks before each run.
self.run_suite_mock.reset_mock()
self.warning_mock.reset_mock()
self.failure_mock.reset_mock()
to_raise = None
if cmd_fail_mode == None:
to_raise = None
elif cmd_fail_mode == 'timeout':
to_raise = timeout_util.TimeoutError('Timed out')
elif cmd_fail_mode == 'suite_timeout':
to_raise = failures_lib.SuiteTimedOut('Suite timed out')
elif cmd_fail_mode == 'board_not_available':
to_raise = failures_lib.BoardNotAvailable('Board not available')
elif cmd_fail_mode == 'lab_fail':
to_raise = failures_lib.TestLabFailure('Test lab failure')
elif cmd_fail_mode == 'test_warn':
to_raise = failures_lib.TestWarning('Suite passed with warnings')
elif cmd_fail_mode == 'test_fail':
to_raise = failures_lib.TestFailure('HWTest failed.')
else:
raise ValueError('cmd_fail_mode %s not supported' % cmd_fail_mode)
if cmd_fail_mode == 'timeout':
self.run_suite_mock.side_effect = to_raise
else:
self.run_suite_mock.return_value = commands.HWTestSuiteResult(
to_raise, None)
if fails:
self.assertRaises(failures_lib.StepFailure, self.RunStage)
else:
self.RunStage()
self.run_suite_mock.assert_called_once()
self.assertEqual(self.run_suite_mock.call_args[1].get('debug'), debug)
# Make sure we print the buildbot failure/warning messages correctly.
if fails:
self.failure_mock.assert_called_once()
else:
self.assertFalse(self.failure_mock.called)
if warns:
self.warning_mock.assert_called_once()
else:
self.assertFalse(self.warning_mock.called)
def RunTestSuite(buildroot,
board,
image_path,
results_dir,
test_config,
whitelist_chrome_crashes,
archive_dir,
ssh_private_key=None,
ssh_port=9228):
"""Runs the test harness suite."""
results_dir_in_chroot = os.path.join(buildroot, 'chroot',
results_dir.lstrip('/'))
osutils.RmDir(results_dir_in_chroot, ignore_missing=True)
test_type = test_config.test_type
cwd = os.path.join(buildroot, 'src', 'scripts')
dut_type = 'gce' if test_type == constants.GCE_SUITE_TEST_TYPE else 'vm'
cmd = [
'bin/ctest',
'--board=%s' % board,
'--type=%s' % dut_type, '--no_graphics', '--verbose',
'--target_image=%s' % image_path,
'--test_results_root=%s' % results_dir_in_chroot
]
if test_type not in constants.VALID_VM_TEST_TYPES:
raise AssertionError('Unrecognized test type %r' % test_type)
if test_type == constants.FULL_AU_TEST_TYPE:
cmd.append('--archive_dir=%s' % archive_dir)
elif test_type in [
constants.VM_SUITE_TEST_TYPE, constants.GCE_SUITE_TEST_TYPE
]:
cmd.append('--ssh_port=%s' % ssh_port)
cmd.append('--only_verify')
cmd.append('--suite=%s' % test_config.test_suite)
else:
cmd.append('--quick_update')
if whitelist_chrome_crashes:
cmd.append('--whitelist_chrome_crashes')
if ssh_private_key is not None:
cmd.append('--ssh_private_key=%s' % ssh_private_key)
# Give tests 10 minutes to clean up before shutting down.
result = cros_build_lib.RunCommand(cmd,
cwd=cwd,
error_code_ok=True,
kill_timeout=10 * 60)
if result.returncode:
if os.path.exists(results_dir_in_chroot):
error = '%s exited with code %d' % (' '.join(cmd),
result.returncode)
with open(results_dir_in_chroot + '/failed_test_command',
'w') as failed:
failed.write(error)
raise failures_lib.TestFailure('** VMTests failed with code %d **' %
result.returncode)
