def extract_test_data(self):
"""Extracts data emitted by the test."""
if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
file_util.move_raw_coverage_data(self.udid, self.out_dir)
# Find the Documents directory of the test app. The app directory names
# don't correspond with any known information, so we have to examine them
# all until we find one with a matching CFBundleIdentifier.
apps_dir = os.path.join(
self.homedir, 'Containers', 'Data', 'Application')
if os.path.exists(apps_dir):
for appid_dir in os.listdir(apps_dir):
docs_dir = os.path.join(apps_dir, appid_dir, 'Documents')
metadata_plist = os.path.join(
apps_dir,
appid_dir,
'.com.apple.mobile_container_manager.metadata.plist',
)
if os.path.exists(docs_dir) and os.path.exists(metadata_plist):
cfbundleid = subprocess.check_output([
'/usr/libexec/PlistBuddy',
'-c', 'Print:MCMMetadataIdentifier',
metadata_plist,
]).rstrip()
if cfbundleid == self.cfbundleid:
shutil.copytree(docs_dir, os.path.join(self.out_dir, 'Documents'))
return
def test_move_raw_coverage_data(self): """Tests if file_util can correctly move raw coverage data""" self.create_origin_profraw_file_if_not_exist() self.assertTrue(os.path.exists(self.origin_profraw_file_path)) self.assertFalse(os.path.exists(self.expected_profraw_output_path)) file_util.move_raw_coverage_data(self.existing_udid, self.output_folder) self.assertFalse(os.path.exists(self.origin_profraw_file_path)) self.assertTrue(os.path.exists(self.expected_profraw_output_path)) os.remove(self.expected_profraw_output_path)
def test_move_raw_coverage_data_origin_not_exist(self):
"""Ensures that file_util won't break when raw coverage data folder or
file doesn't exist
"""
# Tests origin directory doesn't exist.
file_util.move_raw_coverage_data(self.not_existing_udid, self.output_folder)
self.assertFalse(os.path.exists(self.expected_profraw_output_path))
# Tests profraw file doesn't exist.
if os.path.exists(self.origin_profraw_file_path):
os.remove(self.origin_profraw_file_path)
self.assertFalse(os.path.exists(self.origin_profraw_file_path))
self.assertFalse(os.path.exists(self.expected_profraw_output_path))
file_util.move_raw_coverage_data(self.existing_udid, self.output_folder)
self.assertFalse(os.path.exists(self.expected_profraw_output_path))
def launch(self):
"""Launches tests using xcodebuild."""
self.test_results['attempts'] = []
cancelled_statuses = {'TESTS_DID_NOT_START', 'BUILD_INTERRUPTED'}
shards = self.shards
running_tests = set(self.egtests_app.get_all_tests())
# total number of attempts is self.retries+1
for attempt in range(self.retries + 1):
# Erase all simulators per each attempt
if iossim_util.is_device_with_udid_simulator(self.udid):
# kill all running simulators to prevent possible memory leaks
test_runner.SimulatorTestRunner.kill_simulators()
shutdown_all_simulators()
shutdown_all_simulators(XTDEVICE_FOLDER)
erase_all_simulators()
erase_all_simulators(XTDEVICE_FOLDER)
outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt)
cmd_list = self.egtests_app.command(outdir_attempt,
'id=%s' % self.udid, shards)
# TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner.
LOGGER.info('Start test attempt #%d for command [%s]' %
(attempt, ' '.join(cmd_list)))
output = self.launch_attempt(cmd_list)
if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
# out_dir of LaunchCommand object is the TestRunner out_dir joined with
# UDID. Use os.path.dirname to retrieve the TestRunner out_dir.
file_util.move_raw_coverage_data(self.udid,
os.path.dirname(self.out_dir))
self.test_results['attempts'].append(
self._log_parser.collect_test_results(outdir_attempt, output))
# Do not exit here when no failed test from parsed log and parallel
# testing is enabled (shards > 1), because when one of the shards fails
# before tests start , the tests not run don't appear in log at all.
if (self.retries == attempt
or (shards == 1
and not self.test_results['attempts'][-1]['failed'])):
break
# Exclude passed tests in next test attempt.
self.egtests_app.excluded_tests += self.test_results['attempts'][
-1]['passed']
# crbug.com/987664 - for the case when
# all tests passed but build was interrupted,
# excluded(passed) tests are equal to tests to run.
if set(self.egtests_app.excluded_tests) == running_tests:
for status in cancelled_statuses:
failure = self.test_results['attempts'][-1]['failed'].pop(
status, None)
if failure:
LOGGER.info('Failure for passed tests %s: %s' %
(status, failure))
break
# If tests are not completed(interrupted or did not start)
# re-run them with the same number of shards,
# otherwise re-run with shards=1 and exclude passed tests.
cancelled_attempt = cancelled_statuses.intersection(
self.test_results['attempts'][-1]['failed'].keys())
# Item in cancelled_statuses is used to config for next attempt. The usage
# should be confined in this method. Real tests affected by these statuses
# will be marked timeout in results.
for status in cancelled_statuses:
self.test_results['attempts'][-1]['failed'].pop(status, None)
if (not cancelled_attempt
# If need to re-run less than 20 tests, 1 shard should be enough.
or
(len(running_tests) - len(self.egtests_app.excluded_tests) <=
MAXIMUM_TESTS_PER_SHARD_FOR_RERUN)):
shards = 1
self.summary_log()
return {'test_results': self.test_results, 'logs': self.logs}
def launch(self):
"""Launches tests using xcodebuild."""
overall_launch_command_result = ResultCollection()
shards = self.shards
running_tests = set(self.egtests_app.get_all_tests())
# total number of attempts is self.retries+1
for attempt in range(self.retries + 1):
# Erase all simulators per each attempt
if iossim_util.is_device_with_udid_simulator(self.udid):
# kill all running simulators to prevent possible memory leaks
test_runner.SimulatorTestRunner.kill_simulators()
shutdown_all_simulators()
shutdown_all_simulators(XTDEVICE_FOLDER)
erase_all_simulators()
erase_all_simulators(XTDEVICE_FOLDER)
outdir_attempt = os.path.join(self.out_dir, 'attempt_%d' % attempt)
cmd_list = self.egtests_app.command(outdir_attempt,
'id=%s' % self.udid, shards)
# TODO(crbug.com/914878): add heartbeat logging to xcodebuild_runner.
LOGGER.info('Start test attempt #%d for command [%s]' %
(attempt, ' '.join(cmd_list)))
output = self.launch_attempt(cmd_list)
if hasattr(self, 'use_clang_coverage') and self.use_clang_coverage:
# out_dir of LaunchCommand object is the TestRunner out_dir joined with
# UDID. Use os.path.dirname to retrieve the TestRunner out_dir.
file_util.move_raw_coverage_data(self.udid,
os.path.dirname(self.out_dir))
result = self._log_parser.collect_test_results(
outdir_attempt, output)
tests_selected_at_runtime = _tests_decided_at_runtime(
self.egtests_app.test_app_path)
# For most suites, only keep crash status from last attempt since retries
# will cover any missing tests. For these decided at runtime, retain
# crashes from all attempts and a dummy "crashed" result will be reported
# to indicate some tests might never ran.
# TODO(crbug.com/1235871): Switch back to excluded tests and set
# |overall_crash| to always True.
overall_launch_command_result.add_result_collection(
result, overwrite_crash=not tests_selected_at_runtime)
result.report_to_result_sink()
tests_to_include = set()
# |running_tests| are compiled tests in target intersecting with swarming
# sharding. For some suites, they are more than what's needed to run.
if not tests_selected_at_runtime:
tests_to_include = tests_to_include | (
running_tests -
overall_launch_command_result.expected_tests())
# Add failed tests from last rounds for runtime decided suites and device
# suites.
tests_to_include = (
tests_to_include
| overall_launch_command_result.never_expected_tests())
self.egtests_app.included_tests = list(tests_to_include)
# Nothing to run in retry.
if not self.egtests_app.included_tests:
break
# If tests are not completed(interrupted or did not start) and there are
# >= 20 remaining tests, run them with the same number of shards.
# otherwise re-run with shards=1.
if (not result.crashed
# If need to re-run less than 20 tests, 1 shard should be enough.
or (len(running_tests) -
len(overall_launch_command_result.expected_tests()) <=
MAXIMUM_TESTS_PER_SHARD_FOR_RERUN)):
shards = 1
return overall_launch_command_result