def testGetTestStatuses(self, mock_xcresult, mock_export): mock_xcresult.side_effect = _xcresulttool_get_side_effect # self.assertEqual(test_result.test_log, lo expected_failure_log = ( 'Logs from "failureSummaries" in .xcresult:\n' 'file: /../../ios/web/shell/test/page_state_egtest.mm, line: 131\n' 'Some logs.\n' 'file: , line: \n' 'Immediately halt execution of testcase ' '(EarlGreyInternalTestInterruptException)\n') expected_expected_tests = set( ['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2']) results = xcode_log_parser.Xcode11LogParser()._get_test_statuses( OUTPUT_PATH) self.assertEqual(expected_expected_tests, results.expected_tests()) seen_failed_test = False for test_result in results.test_results: if (test_result.name == 'PageStateTestCase/testZeroContentOffsetAfterLoad'): seen_failed_test = True self.assertEqual(test_result.test_log, expected_failure_log) crash_file_name = 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_1.crash' jpeg_file_name = 'attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_2.jpeg' self.assertDictEqual( { crash_file_name: '/tmp/%s' % crash_file_name, jpeg_file_name: '/tmp/%s' % jpeg_file_name, }, test_result.attachments) self.assertTrue(seen_failed_test)
def __init__(self, egtests_app, destination, shards, retries, out_dir=os.path.basename(os.getcwd()), env=None): """Initialize launch command. Args: egtests_app: (EgtestsApp) An egtests_app to run. destination: (str) A destination. shards: (int) A number of shards. retries: (int) A number of retries. out_dir: (str) A folder in which xcodebuild will generate test output. By default it is a current directory. env: (dict) Environment variables. Raises: LaunchCommandCreationError: if one of parameters was not set properly. """ if not isinstance(egtests_app, EgtestsApp): raise test_runner.AppNotFoundError( 'Parameter `egtests_app` is not EgtestsApp: %s' % egtests_app) self.egtests_app = egtests_app self.destination = destination self.shards = shards self.retries = retries self.out_dir = out_dir self.logs = collections.OrderedDict() self.test_results = collections.OrderedDict() self.env = env if distutils.version.LooseVersion('11.0') <= distutils.version.LooseVersion( test_runner.get_current_xcode_info()['version']): self._log_parser = xcode_log_parser.Xcode11LogParser() else: self._log_parser = xcode_log_parser.XcodeLogParser()
def testCollectTestTesults(self, mock_get_failed_tests, mock_root, mock_exist_file, *args): metrics_json = """ { "metrics": { "testsCount": { "_value": "7" }, "testsFailedCount": { "_value": "14" } } }""" expected_test_results = { 'passed': ['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2'], 'failed': { 'WebUITestCase/testBackForwardFromWebURL': [ 'file://<unknown>#CharacterRangeLen=0', 'Test crashed in <external symbol>' ] } } mock_get_failed_tests.return_value = expected_test_results['failed'] mock_root.side_effect = _xcresulttool_get_side_effect mock_exist_file.return_value = True self.assertEqual( expected_test_results, xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []))
def testCollectTestTesults(self, mock_get_passed_tests, mock_get_failed_tests, mock_root, mock_exist_file): metrics_json = """ { "metrics": { "testsCount": { "_value": "7" }, "testsFailedCount": { "_value": "14" } } }""" expected_test_results = { 'passed': ['TestCase1/testMethod1', 'TestCase2/testMethod1'], 'failed': { 'WebUITestCase/testBackForwardFromWebURL': [ 'file://<unknown>#CharacterRangeLen=0', 'Test crashed in <external symbol>' ] } } mock_get_passed_tests.return_value = expected_test_results['passed'] mock_get_failed_tests.return_value = expected_test_results['failed'] mock_root.return_value = metrics_json mock_exist_file.return_value = True self.assertEqual( expected_test_results, xcode_log_parser.Xcode11LogParser().collect_test_results( _XTEST_RESULT, []))
def testXcresulttoolListPassedTests(self, mock_xcresult): mock_xcresult.return_value = PASSED_TESTS expected = ['TestCase1/testMethod1', 'TestCase2/testMethod1'] self.assertEqual( expected, xcode_log_parser.Xcode11LogParser()._list_of_passed_tests( _XTEST_RESULT))
def testXcresulttoolGetRoot(self, mock_process): mock_process.return_value = '%JSON%' xcode_log_parser.Xcode11LogParser()._xcresulttool_get('xcresult_path') self.assertTrue( os.path.join(XCODE11_DICT['path'], 'usr', 'bin') in os.environ['PATH']) self.assertEqual( ['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'], mock_process.mock_calls[0][1][0])
def testXcresulttoolListPassedTests(self, mock_xcresult): mock_xcresult.side_effect = _xcresulttool_get_side_effect expected = [ 'PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2' ] results = {'passed': [], 'failed': {}} xcode_log_parser.Xcode11LogParser()._get_test_statuses(OUTPUT_PATH, results) self.assertEqual(expected, results['passed'])
def testCollectTestsDidNotRun(self, mock_exist_file): mock_exist_file.return_value = False results = xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []) self.assertTrue(results.crashed) self.assertEqual(results.crash_message, '/tmp/attempt_0 with staging data does not exist.\n') self.assertEqual(len(results.all_test_names()), 0)
def testXcresulttoolListFailedTests(self): failure_message = [ 'file://<unknown>#CharacterRangeLen=0' ] + 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}'.splitlines() expected = {'WebUITestCase/testBackForwardFromWebURL': failure_message} self.assertEqual( expected, xcode_log_parser.Xcode11LogParser()._list_of_failed_tests( json.loads(ACTIONS_RECORD_FAILED_TEST)))
def testCollectTestsInterruptedRun(self, mock_exist_file): mock_exist_file.side_effect = [True, False] results = xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []) self.assertTrue(results.crashed) self.assertEqual( results.crash_message, '/tmp/attempt_0.xcresult with test results does not exist.\n') self.assertEqual(len(results.all_test_names()), 0)
def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file, *args): metrics_json = '{"metrics": {}}' mock_root.return_value = metrics_json mock_exist_file.return_value = True results = xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []) self.assertTrue(results.crashed) self.assertEqual(results.crash_message, '0 tests executed!') self.assertEqual(len(results.all_test_names()), 0)
def testArtifactsDiagnosticLogsExportedInCollectTestTesults( self, mock_get_failed_tests, mock_root, mock_exist_file, mock_export_diagnostic_data, mock_extract_artifacts, mock_zip): mock_root.side_effect = _xcresulttool_get_side_effect mock_exist_file.return_value = True xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []) mock_export_diagnostic_data.assert_called_with(OUTPUT_PATH) mock_extract_artifacts.assert_called()
def testCollectTestsDidNotRun(self, mock_exist_file): mock_exist_file.return_value = False expected_test_results = { 'passed': [], 'failed': {'TESTS_DID_NOT_START': [ '%s with test results does not exist.' % _XTEST_RESULT]}} self.assertEqual(expected_test_results, xcode_log_parser.Xcode11LogParser().collect_test_results( _XTEST_RESULT))
def testCollectTestsInterruptedRun(self, mock_exist_file): mock_exist_file.side_effect = [True, False] expected_test_results = { 'passed': [], 'failed': {'BUILD_INTERRUPTED': [ '%s with test results does not exist.' % os.path.join( _XTEST_RESULT + '.xcresult', 'Info.plist')]}} self.assertEqual(expected_test_results, xcode_log_parser.Xcode11LogParser().collect_test_results( _XTEST_RESULT))
def testXcresulttoolGetRef(self, mock_process): mock_process.side_effect = [REF_ID, 'JSON'] xcode_log_parser.Xcode11LogParser()._xcresulttool_get('xcresult_path', 'testsRef') self.assertEqual( ['xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path'], mock_process.mock_calls[0][1][0]) self.assertEqual([ 'xcresulttool', 'get', '--format', 'json', '--path', 'xcresult_path', '--id', 'REF_ID'], mock_process.mock_calls[1][1][0])
def testCollectTestsRanZeroTests(self, mock_root, mock_exist_file): metrics_json = '{"metrics": {}}' expected_test_results = { 'passed': [], 'failed': {'TESTS_DID_NOT_START': ['0 tests executed!']}} mock_root.return_value = metrics_json mock_exist_file.return_value = True self.assertEqual(expected_test_results, xcode_log_parser.Xcode11LogParser().collect_test_results( _XTEST_RESULT))
def testXcresulttoolListFailedTests(self): failure_message = ( 'file:///../../ios/web/shell/test/page_state_egtest.mm#' 'CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130\n' 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}') expected = set(['PageStateTestCase/testZeroContentOffsetAfterLoad']) results = xcode_log_parser.Xcode11LogParser()._list_of_failed_tests( json.loads(XCRESULT_ROOT)) self.assertEqual(expected, results.failed_tests()) log = results.test_results[0].test_log self.assertEqual(log, failure_message)
def testXcresulttoolListFailedTests(self): failure_message = [ 'file:///../../ios/web/shell/test/page_state_egtest.mm#' 'CharacterRangeLen=0&EndingLineNumber=130&StartingLineNumber=130' ] + 'Fail. Screenshots: {\n\"Failure\": \"path.png\"\n}'.splitlines() expected = { 'PageStateTestCase/testZeroContentOffsetAfterLoad': failure_message } self.assertEqual( expected, xcode_log_parser.Xcode11LogParser()._list_of_failed_tests( json.loads(XCRESULT_ROOT)))
def testCollectTestsDidNotRun(self, mock_exist_file): mock_exist_file.return_value = False expected_test_results = { 'passed': [], 'failed': { 'TESTS_DID_NOT_START': ['%s with staging data does not exist.' % OUTPUT_PATH] } } self.assertEqual( expected_test_results, xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []))
def testCollectTestsInterruptedRun(self, mock_exist_file): mock_exist_file.side_effect = [True, False] expected_test_results = { 'passed': [], 'failed': { 'BUILD_INTERRUPTED': ['%s with test results does not exist.' % XCRESULT_PATH] } } self.assertEqual( expected_test_results, xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []))
def testCollectTestResults_interruptedTests(self, mock_path_exists): mock_path_exists.side_effect = [True, False] output = [ '[09:03:42:INFO] Test case \'-[TestCase1 method1]\' passed on device.', '[09:06:40:INFO] Test Case \'-[TestCase2 method1]\' passed on device.', '[09:09:00:INFO] Test case \'-[TestCase2 method1]\' failed on device.', '** BUILD INTERRUPTED **', ] not_found_message = [ 'Info.plist.xcresult/Info.plist with test results does not exist.'] res = xcode_log_parser.Xcode11LogParser().collect_test_results( 'Info.plist', output) self.assertIn('BUILD_INTERRUPTED', res['failed']) self.assertEqual(not_found_message + output, res['failed']['BUILD_INTERRUPTED']) self.assertEqual(['TestCase1/method1', 'TestCase2/method1'], res['passed'])
def testCollectTestResults_interruptedTests(self, mock_path_exists): mock_path_exists.side_effect = [True, False] output = [ '[09:03:42:INFO] Test case \'-[TestCase1 method1]\' passed on device.', '[09:06:40:INFO] Test Case \'-[TestCase2 method1]\' passed on device.', '[09:09:00:INFO] Test case \'-[TestCase2 method1]\' failed on device.', '** BUILD INTERRUPTED **', ] not_found_message = [ '%s with test results does not exist.' % XCRESULT_PATH ] res = xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, output) self.assertTrue(res.crashed) self.assertEqual('\n'.join(not_found_message + output), res.crash_message) self.assertEqual(set(['TestCase1/method1', 'TestCase2/method1']), res.expected_tests())
def testCopyScreenshots(self, mock_xcresulttool_get, mock_path_exists, mock_process): mock_path_exists.return_value = True mock_xcresulttool_get.side_effect = _xcresulttool_get_side_effect xcode_log_parser.Xcode11LogParser().copy_artifacts(OUTPUT_PATH) mock_process.assert_any_call([ 'xcresulttool', 'export', '--type', 'file', '--id', 'SCREENSHOT_REF_ID_IN_FAILURE_SUMMARIES', '--path', XCRESULT_PATH, '--output-path', '/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_2.jpeg' ]) mock_process.assert_any_call([ 'xcresulttool', 'export', '--type', 'file', '--id', 'CRASH_REF_ID_IN_ACTIVITY_SUMMARIES', '--path', XCRESULT_PATH, '--output-path', '/tmp/attempt_0_PageStateTestCase_testZeroContentOffsetAfterLoad_1' '.crash' ]) # Ensures screenshots in activitySummaries are not copied. self.assertEqual(2, mock_process.call_count)
def setUp(self): super(DeviceXcodeTestRunnerTest, self).setUp() self.mock(os.path, 'exists', lambda _: True) self.mock( test_runner, 'get_current_xcode_info', lambda: { 'version': 'test version', 'build': 'test build', 'path': 'test/path' }) self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path) self.mock(result_sink_util.ResultSinkClient, 'post', lambda *args, **kwargs: None) self.mock(test_runner.subprocess, 'check_output', lambda _: 'fake-output') self.mock(test_runner.subprocess, 'check_call', lambda _: 'fake-out') self.mock(test_runner.subprocess, 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') self.mock(test_runner.TestRunner, 'set_sigterm_handler', lambda self, handler: 0) self.mock(os, 'listdir', lambda _: []) self.mock(xcodebuild_runner.subprocess, 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') self.mock(test_runner, 'print_process_output', lambda _: []) self.mock(test_runner.TestRunner, 'start_proc', lambda self, cmd: 0) self.mock(test_runner.DeviceTestRunner, 'get_installed_packages', lambda self: []) self.mock(test_runner.DeviceTestRunner, 'wipe_derived_data', lambda _: None) self.mock(test_runner.TestRunner, 'retrieve_derived_data', lambda _: None) self.mock(test_runner.TestRunner, 'process_xcresult_dir', lambda _: None) self.mock(xcode_log_parser, 'get_parser', lambda: xcode_log_parser.Xcode11LogParser()) self.mock(test_apps.EgtestsApp, 'fill_xctest_run', lambda _1, _2: 'xctestrun') self.mock(test_apps.GTestsApp, 'get_all_tests', lambda _: ['Class1/passedTest1', 'Class1/passedTest2']) self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False)
def testCollectTestTesults(self, mock_root, mock_exist_file, *args): expected_passed = set( ['PageStateTestCase/testMethod1', 'PageStateTestCase/testMethod2']) expected_failed = set( ['PageStateTestCase/testZeroContentOffsetAfterLoad']) mock_root.side_effect = _xcresulttool_get_side_effect mock_exist_file.return_value = True results = xcode_log_parser.Xcode11LogParser().collect_test_results( OUTPUT_PATH, []) # Length ensures no duplicate results from |_get_test_statuses| and # |_list_of_failed_tests|. self.assertEqual(len(results.test_results), 3) self.assertEqual(expected_passed, results.expected_tests()) self.assertEqual(expected_failed, results.unexpected_tests()) # Ensure format. for test in results.test_results: self.assertTrue(isinstance(test.name, str)) if test.status == TestStatus.FAIL: self.assertTrue(isinstance(test.test_log, str))
def setUp(self): super(XCodebuildRunnerTest, self).setUp() self.mock(os.path, 'exists', lambda _: True) self.mock(xcode_log_parser, 'get_parser', lambda: xcode_log_parser.Xcode11LogParser()) self.mock(os, 'listdir', lambda _: ['any_egtests.xctest']) self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False) self.mock(result_sink_util.ResultSinkClient, 'post', lambda *args, **kwargs: None) self.mock(test_apps.GTestsApp, 'get_all_tests', lambda _: ['Class1/passedTest1', 'Class1/passedTest2']) self.mock(test_apps.EgtestsApp, 'fill_xctest_run', lambda _1, _2: 'xctestrun') self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID') self.mock(test_apps, 'get_bundle_id', lambda _: "fake-bundle-id") self.mock(test_apps, 'is_running_rosetta', lambda: False) self.mock(test_apps.plistlib, 'writePlist', lambda _1, _2: '') self.mock(test_runner.SimulatorTestRunner, 'tear_down', lambda _: None) self.mock(test_runner.DeviceTestRunner, 'tear_down', lambda _: None) self.mock(xcodebuild_runner.subprocess, 'Popen', lambda cmd, env, stdout, stderr: 'fake-out') self.mock(test_runner, 'print_process_output', lambda _: [])
def testXcresulttoolListFailedTestsExclude(self): excluded = set(['PageStateTestCase/testZeroContentOffsetAfterLoad']) results = xcode_log_parser.Xcode11LogParser()._list_of_failed_tests( json.loads(XCRESULT_ROOT), excluded=excluded) self.assertEqual(set([]), results.all_test_names())
def testCopyScreenshots(self, mock_copy, mock_xcresulttool_get, mock_exist_file): mock_exist_file.return_value = True mock_xcresulttool_get.return_value = ACTIONS_RECORD_FAILED_TEST xcode_log_parser.Xcode11LogParser().copy_screenshots(_XTEST_RESULT) self.assertEqual(1, mock_copy.call_count)