def testGetTestStatuses(self, mock_xcresult, mock_export):
mock_xcresult.side_effect = _xcresulttool_get_side_effect
# self.assertEqual(test_result.test_log, lo
expected_failure_log = (
'Logs from "failureSummaries" in .xcresult:\n'
'file: /../../ios/web/shell/test/page_state_egtest.mm, line: 131\n'
'Some logs.\n'
'file: , line: \n'
'Immediately halt execution of testcase '
'(EarlGreyInternalTestInterruptException)\n')
expected_expected_tests = set(
['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'])
results = xcode_log_parser.Xcode11LogParser()._get_test_statuses(
OUTPUT_PATH)
self.assertEqual(expected_expected_tests, results.expected_tests())
seen_failed_test = False
for test_result in results.test_results:
if (test_result.name ==
'PageStateTestCase/testZeroContentOffsetAfterLoad'):
seen_failed_test = True
self.assertEqual(test_result.test_log, expected_failure_log)
crash_file_name = 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_1.crash'
jpeg_file_name = 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_2.jpeg'
self.assertDictEqual(
{
crash_file_name: '/tmp/%s' % crash_file_name,
jpeg_file_name: '/tmp/%s' % jpeg_file_name,
}, test_result.attachments)
self.assertTrue(seen_failed_test)
def __init__(self, egtests_app, destination,
shards,
retries,
out_dir=os.path.basename(os.getcwd()),
env=None):
"""Initialize launch command.
Args:
egtests_app: (EgtestsApp) An egtests_app to run.
destination: (str) A destination.
shards: (int) A number of shards.
retries: (int) A number of retries.
out_dir: (str) A folder in which xcodebuild will generate test output.
By default it is a current directory.
env: (dict) Environment variables.
Raises:
LaunchCommandCreationError: if one of parameters was not set properly.
"""
if not isinstance(egtests_app, EgtestsApp):
raise test_runner.AppNotFoundError(
'Parameter `egtests_app` is not EgtestsApp: %s' % egtests_app)
self.egtests_app = egtests_app
self.destination = destination
self.shards = shards
self.retries = retries
self.out_dir = out_dir
self.logs = collections.OrderedDict()
self.test_results = collections.OrderedDict()
self.env = env
if distutils.version.LooseVersion('11.0') <= distutils.version.LooseVersion(
test_runner.get_current_xcode_info()['version']):
self._log_parser = xcode_log_parser.Xcode11LogParser()
else:
self._log_parser = xcode_log_parser.XcodeLogParser()
def testCollectTestTesults(self, mock_get_failed_tests, mock_root,
mock_exist_file, *args):
metrics_json = """
{
"metrics": {
"testsCount": {
"_value": "7"
},
"testsFailedCount": {
"_value": "14"
}
}
}"""
expected_test_results = {
'passed':
['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'],
'failed': {
'WebUITestCase/testBackForwardFromWebURL': [
'file://<unknown>#CharacterRangeLen=0',
'Test crashed in <external symbol>'
]
}
}
mock_get_failed_tests.return_value = expected_test_results['failed']
mock_root.side_effect = _xcresulttool_get_side_effect
mock_exist_file.return_value = True
self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, []))
def testCollectTestTesults(self, mock_get_passed_tests,
mock_get_failed_tests, mock_root,
mock_exist_file):
metrics_json = """
{
"metrics": {
"testsCount": {
"_value": "7"
},
"testsFailedCount": {
"_value": "14"
}
}
}"""
expected_test_results = {
'passed': ['TestCase1/testMethod1', 'TestCase2/testMethod1'],
'failed': {
'WebUITestCase/testBackForwardFromWebURL': [
'file://<unknown>#CharacterRangeLen=0',
'Test crashed in <external symbol>'
]
}
}
mock_get_passed_tests.return_value = expected_test_results['passed']
mock_get_failed_tests.return_value = expected_test_results['failed']
mock_root.return_value = metrics_json
mock_exist_file.return_value = True
self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT, []))
def testXcresulttoolListPassedTests(self, mock_xcresult):
mock_xcresult.return_value = PASSED_TESTS
expected = ['TestCase1/testMethod1', 'TestCase2/testMethod1']
self.assertEqual(
expected,
xcode_log_parser.Xcode11LogParser()._list_of_passed_tests(
_XTEST_RESULT))
def testXcresulttoolGetRoot(self, mock_process):
mock_process.return_value = '%JSON%'
xcode_log_parser.Xcode11LogParser()._xcresulttool_get('xcresult_path')
self.assertTrue(
os.path.join(XCODE11_DICT['path'], 'usr', 'bin') in os.environ['PATH'])
self.assertEqual(
['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'],
mock_process.mock_calls[0][1][0])
def testXcresulttoolListPassedTests(self, mock_xcresult):
mock_xcresult.side_effect = _xcresulttool_get_side_effect
expected = [
'PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'
]
results = {'passed': [], 'failed': {}}
xcode_log_parser.Xcode11LogParser()._get_test_statuses(OUTPUT_PATH, results)
self.assertEqual(expected, results['passed'])
def testCollectTestsDidNotRun(self, mock_exist_file):
mock_exist_file.return_value = False
results = xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, [])
self.assertTrue(results.crashed)
self.assertEqual(results.crash_message,
'/tmp/attempt_0 with staging data does not exist.\n')
self.assertEqual(len(results.all_test_names()), 0)
def testXcresulttoolListFailedTests(self):
failure_message = [
'file://<unknown>#CharacterRangeLen=0'
] + 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}'.splitlines()
expected = {'WebUITestCase/testBackForwardFromWebURL': failure_message}
self.assertEqual(
expected,
xcode_log_parser.Xcode11LogParser()._list_of_failed_tests(
json.loads(ACTIONS_RECORD_FAILED_TEST)))
def testCollectTestsInterruptedRun(self, mock_exist_file):
mock_exist_file.side_effect = [True, False]
results = xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, [])
self.assertTrue(results.crashed)
self.assertEqual(
results.crash_message,
'/tmp/attempt_0.xcresult with test results does not exist.\n')
self.assertEqual(len(results.all_test_names()), 0)
def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file, *args):
metrics_json = '{"metrics": {}}'
mock_root.return_value = metrics_json
mock_exist_file.return_value = True
results = xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, [])
self.assertTrue(results.crashed)
self.assertEqual(results.crash_message, '0 tests executed!')
self.assertEqual(len(results.all_test_names()), 0)
def testArtifactsDiagnosticLogsExportedInCollectTestTesults(
self, mock_get_failed_tests, mock_root, mock_exist_file,
mock_export_diagnostic_data, mock_extract_artifacts, mock_zip):
mock_root.side_effect = _xcresulttool_get_side_effect
mock_exist_file.return_value = True
xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, [])
mock_export_diagnostic_data.assert_called_with(OUTPUT_PATH)
mock_extract_artifacts.assert_called()
def testCollectTestsDidNotRun(self, mock_exist_file):
mock_exist_file.return_value = False
expected_test_results = {
'passed': [],
'failed': {'TESTS_DID_NOT_START': [
'%s with test results does not exist.' % _XTEST_RESULT]}}
self.assertEqual(expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT))
def testCollectTestsInterruptedRun(self, mock_exist_file):
mock_exist_file.side_effect = [True, False]
expected_test_results = {
'passed': [],
'failed': {'BUILD_INTERRUPTED': [
'%s with test results does not exist.' % os.path.join(
_XTEST_RESULT + '.xcresult', 'Info.plist')]}}
self.assertEqual(expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT))
def testXcresulttoolGetRef(self, mock_process):
mock_process.side_effect = [REF_ID, 'JSON']
xcode_log_parser.Xcode11LogParser()._xcresulttool_get('xcresult_path',
'testsRef')
self.assertEqual(
['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'],
mock_process.mock_calls[0][1][0])
self.assertEqual([
'xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path',
'--id', 'REF_ID'], mock_process.mock_calls[1][1][0])
def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file):
metrics_json = '{"metrics": {}}'
expected_test_results = {
'passed': [],
'failed': {'TESTS_DID_NOT_START': ['0 tests executed!']}}
mock_root.return_value = metrics_json
mock_exist_file.return_value = True
self.assertEqual(expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results(
_XTEST_RESULT))
def testXcresulttoolListFailedTests(self):
failure_message = (
'file:///../../ios/web/shell/test/page_state_egtest.mm#'
'CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130\n'
'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}')
expected = set(['PageStateTestCase/testZeroContentOffsetAfterLoad'])
results = xcode_log_parser.Xcode11LogParser()._list_of_failed_tests(
json.loads(XCRESULT_ROOT))
self.assertEqual(expected, results.failed_tests())
log = results.test_results[0].test_log
self.assertEqual(log, failure_message)
def testXcresulttoolListFailedTests(self):
failure_message = [
'file:///../../ios/web/shell/test/page_state_egtest.mm#'
'CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130'
] + 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}'.splitlines()
expected = {
'PageStateTestCase/testZeroContentOffsetAfterLoad': failure_message
}
self.assertEqual(
expected,
xcode_log_parser.Xcode11LogParser()._list_of_failed_tests(
json.loads(XCRESULT_ROOT)))
def testCollectTestsDidNotRun(self, mock_exist_file):
mock_exist_file.return_value = False
expected_test_results = {
'passed': [],
'failed': {
'TESTS_DID_NOT_START':
['%s with staging data does not exist.' % OUTPUT_PATH]
}
}
self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, []))
def testCollectTestsInterruptedRun(self, mock_exist_file):
mock_exist_file.side_effect = [True, False]
expected_test_results = {
'passed': [],
'failed': {
'BUILD_INTERRUPTED':
['%s with test results does not exist.' % XCRESULT_PATH]
}
}
self.assertEqual(
expected_test_results,
xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, []))
def testCollectTestResults_interruptedTests(self, mock_path_exists):
mock_path_exists.side_effect = [True, False]
output = [
'[09:03:42:INFO] Test case \'-[TestCase1 method1]\' passed on device.',
'[09:06:40:INFO] Test Case \'-[TestCase2 method1]\' passed on device.',
'[09:09:00:INFO] Test case \'-[TestCase2 method1]\' failed on device.',
'** BUILD INTERRUPTED **',
]
not_found_message = [
'Info.plist.xcresult/Info.plist with test results does not exist.']
res = xcode_log_parser.Xcode11LogParser().collect_test_results(
'Info.plist', output)
self.assertIn('BUILD_INTERRUPTED', res['failed'])
self.assertEqual(not_found_message + output,
res['failed']['BUILD_INTERRUPTED'])
self.assertEqual(['TestCase1/method1', 'TestCase2/method1'],
res['passed'])
def testCollectTestResults_interruptedTests(self, mock_path_exists):
mock_path_exists.side_effect = [True, False]
output = [
'[09:03:42:INFO] Test case \'-[TestCase1 method1]\' passed on device.',
'[09:06:40:INFO] Test Case \'-[TestCase2 method1]\' passed on device.',
'[09:09:00:INFO] Test case \'-[TestCase2 method1]\' failed on device.',
'** BUILD INTERRUPTED **',
]
not_found_message = [
'%s with test results does not exist.' % XCRESULT_PATH
]
res = xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, output)
self.assertTrue(res.crashed)
self.assertEqual('\n'.join(not_found_message + output),
res.crash_message)
self.assertEqual(set(['TestCase1/method1', 'TestCase2/method1']),
res.expected_tests())
def testCopyScreenshots(self, mock_xcresulttool_get, mock_path_exists,
mock_process):
mock_path_exists.return_value = True
mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect
xcode_log_parser.Xcode11LogParser().copy_artifacts(OUTPUT_PATH)
mock_process.assert_any_call([
'xcresulttool', 'export', '--type', 'file', '--id',
'SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES', '--path', XCRESULT_PATH,
'--output-path',
'/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_2.jpeg'
])
mock_process.assert_any_call([
'xcresulttool', 'export', '--type', 'file', '--id',
'CRASH_REF_ID_IN_ACTIVITY_SUMMARIES', '--path', XCRESULT_PATH,
'--output-path',
'/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_1'
'.crash'
])
# Ensures screenshots in activitySummaries are not copied.
self.assertEqual(2, mock_process.call_count)
def setUp(self):
super(DeviceXcodeTestRunnerTest, self).setUp()
self.mock(os.path, 'exists', lambda _: True)
self.mock(
test_runner, 'get_current_xcode_info', lambda: {
'version': 'test version',
'build': 'test build',
'path': 'test/path'
})
self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)
self.mock(result_sink_util.ResultSinkClient, 'post',
lambda *args, **kwargs: None)
self.mock(test_runner.subprocess, 'check_output',
lambda _: 'fake-output')
self.mock(test_runner.subprocess, 'check_call', lambda _: 'fake-out')
self.mock(test_runner.subprocess, 'Popen',
lambda cmd, env, stdout, stderr: 'fake-out')
self.mock(test_runner.TestRunner, 'set_sigterm_handler',
lambda self, handler: 0)
self.mock(os, 'listdir', lambda _: [])
self.mock(xcodebuild_runner.subprocess, 'Popen',
lambda cmd, env, stdout, stderr: 'fake-out')
self.mock(test_runner, 'print_process_output', lambda _: [])
self.mock(test_runner.TestRunner, 'start_proc', lambda self, cmd: 0)
self.mock(test_runner.DeviceTestRunner, 'get_installed_packages',
lambda self: [])
self.mock(test_runner.DeviceTestRunner, 'wipe_derived_data',
lambda _: None)
self.mock(test_runner.TestRunner, 'retrieve_derived_data',
lambda _: None)
self.mock(test_runner.TestRunner, 'process_xcresult_dir',
lambda _: None)
self.mock(xcode_log_parser, 'get_parser',
lambda: xcode_log_parser.Xcode11LogParser())
self.mock(test_apps.EgtestsApp, 'fill_xctest_run',
lambda _1, _2: 'xctestrun')
self.mock(test_apps.GTestsApp, 'get_all_tests',
lambda _: ['Class1/passedTest1', 'Class1/passedTest2'])
self.mock(iossim_util, 'is_device_with_udid_simulator',
lambda _: False)
def testCollectTestTesults(self, mock_root, mock_exist_file, *args):
expected_passed = set(
['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'])
expected_failed = set(
['PageStateTestCase/testZeroContentOffsetAfterLoad'])
mock_root.side_effect = _xcresulttool_get_side_effect
mock_exist_file.return_value = True
results = xcode_log_parser.Xcode11LogParser().collect_test_results(
OUTPUT_PATH, [])
# Length ensures no duplicate results from |_get_test_statuses| and
# |_list_of_failed_tests|.
self.assertEqual(len(results.test_results), 3)
self.assertEqual(expected_passed, results.expected_tests())
self.assertEqual(expected_failed, results.unexpected_tests())
# Ensure format.
for test in results.test_results:
self.assertTrue(isinstance(test.name, str))
if test.status == TestStatus.FAIL:
self.assertTrue(isinstance(test.test_log, str))
def setUp(self):
super(XCodebuildRunnerTest, self).setUp()
self.mock(os.path, 'exists', lambda _: True)
self.mock(xcode_log_parser, 'get_parser',
lambda: xcode_log_parser.Xcode11LogParser())
self.mock(os, 'listdir', lambda _: ['any_egtests.xctest'])
self.mock(iossim_util, 'is_device_with_udid_simulator',
lambda _: False)
self.mock(result_sink_util.ResultSinkClient, 'post',
lambda *args, **kwargs: None)
self.mock(test_apps.GTestsApp, 'get_all_tests',
lambda _: ['Class1/passedTest1', 'Class1/passedTest2'])
self.mock(test_apps.EgtestsApp, 'fill_xctest_run',
lambda _1, _2: 'xctestrun')
self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID')
self.mock(test_apps, 'get_bundle_id', lambda _: "fake-bundle-id")
self.mock(test_apps, 'is_running_rosetta', lambda: False)
self.mock(test_apps.plistlib, 'writePlist', lambda _1, _2: '')
self.mock(test_runner.SimulatorTestRunner, 'tear_down', lambda _: None)
self.mock(test_runner.DeviceTestRunner, 'tear_down', lambda _: None)
self.mock(xcodebuild_runner.subprocess, 'Popen',
lambda cmd, env, stdout, stderr: 'fake-out')
self.mock(test_runner, 'print_process_output', lambda _: [])
def testXcresulttoolListFailedTestsExclude(self):
excluded = set(['PageStateTestCase/testZeroContentOffsetAfterLoad'])
results = xcode_log_parser.Xcode11LogParser()._list_of_failed_tests(
json.loads(XCRESULT_ROOT), excluded=excluded)
self.assertEqual(set([]), results.all_test_names())
def testCopyScreenshots(self, mock_copy, mock_xcresulttool_get,
mock_exist_file):
mock_exist_file.return_value = True
mock_xcresulttool_get.return_value = ACTIONS_RECORD_FAILED_TEST
xcode_log_parser.Xcode11LogParser().copy_screenshots(_XTEST_RESULT)
self.assertEqual(1, mock_copy.call_count)
