def _run(self, cmd, shards=1): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. Returns: TestResult.ResultCollection() object. """ parser = gtest_utils.GTestLogParser() # TODO(crbug.com/812705): Implement test sharding for unit tests. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well. proc = self.start_proc(cmd) old_handler = self.set_sigterm_handler( lambda _signum, _frame: self.handle_sigterm(proc)) print_process_output(proc, 'xcodebuild', parser) LOGGER.info('Waiting for test process to terminate.') proc.wait() LOGGER.info('Test process terminated.') self.set_sigterm_handler(old_handler) sys.stdout.flush() LOGGER.debug('Stdout flushed after test process.') returncode = proc.returncode LOGGER.info('%s returned %s\n', cmd[0], returncode) return parser.GetResultCollection()
def _run(self, cmd, shards=1): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. Returns: GTestResult instance. """ result = gtest_utils.GTestResult(cmd) if self.webrtc_xctest: parser = xctest_utils.XCTestLogParser() else: parser = gtest_utils.GTestLogParser() # TODO(crbug.com/812705): Implement test sharding for unit tests. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well. proc = self.start_proc(cmd) old_handler = self.set_sigterm_handler( lambda _signum, _frame: self.handle_sigterm(proc)) print_process_output(proc, 'xcodebuild', parser) LOGGER.info('Waiting for test process to terminate.') proc.wait() LOGGER.info('Test process terminated.') self.set_sigterm_handler(old_handler) sys.stdout.flush() LOGGER.debug('Stdout flushed after test process.') returncode = proc.returncode if self.webrtc_xctest and parser.SystemAlertPresent(): raise SystemAlertPresentError() LOGGER.debug('Processing test results.') for test in parser.FailedTests(include_flaky=True): # Test cases are named as <test group>.<test case>. If the test case # is prefixed with "FLAKY_", it should be reported as flaked not failed. if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[test] = parser.FailureDescription(test) else: result.failed_tests[test] = parser.FailureDescription(test) result.passed_tests.extend(parser.PassedTests(include_flaky=True)) # Only GTest outputs compiled tests in a json file. if not self.webrtc_xctest: result.disabled_tests_from_compiled_tests_file.extend( parser.DisabledTestsFromCompiledTestsFile()) LOGGER.info('%s returned %s\n', cmd[0], returncode) # xcodebuild can return 5 if it exits noncleanly even if all tests passed. # Therefore we cannot rely on process exit code to determine success. result.finalize(returncode, parser.CompletedWithoutFailure()) return result
def _run(self, cmd): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. Returns: GTestResult instance. """ print ' '.join(cmd) print result = gtest_utils.GTestResult(cmd) if self.xctest_path: parser = xctest_utils.XCTestLogParser() else: parser = gtest_utils.GTestLogParser() proc = subprocess.Popen( cmd, env=self.get_launch_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) while True: line = proc.stdout.readline() if not line: break line = line.rstrip() parser.ProcessLine(line) print line sys.stdout.flush() proc.wait() sys.stdout.flush() for test in parser.FailedTests(include_flaky=True): # Test cases are named as <test group>.<test case>. If the test case # is prefixed with "FLAKY_", it should be reported as flaked not failed. if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[test] = parser.FailureDescription(test) else: result.failed_tests[test] = parser.FailureDescription(test) result.passed_tests.extend(parser.PassedTests(include_flaky=True)) print '%s returned %s' % (cmd[0], proc.returncode) print # iossim can return 5 if it exits noncleanly even if all tests passed. # Therefore we cannot rely on process exit code to determine success. result.finalize(proc.returncode, parser.CompletedWithoutFailure()) return result
def run_wpr_test(self, udid, test_name, recipe_path, replay_path): """Runs a single WPR test. Args: udid: UDID for the simulator to run the test on test_name: Test name(format: ios_website) of this wpr test. recipe_path: Path to the recipe file (i.e. ios_costco.test) replay_path: Path to the replay file (i.e. ios_costco) Returns [parser, return code from test] where parser: a XCTest or GTestLogParser which has processed all the output from the test """ LOGGER.info('Running test for recipe %s', recipe_path) self.wprgo_start(replay_path) # TODO(crbug.com/881096): Consider reusing get_launch_command # and adding the autofillautomation flag to it # TODO(crbug.com/881096): We only run AutofillAutomationTestCase # as we have other unit tests in the suite which are not related # to testing website recipe/replays. We should consider moving # one or the other to a different suite. # For the website replay test suite, we need to pass in a single # recipe at a time, with flags "autofillautomation={recipe_path}", # "--enable-features=AutofillShowTypePredictions". The args are written in # xctestrun file, which is produced through EgtestsApp and LaunchCommand # defined in xcodebuild_runner. wpr_test_cmd = self.get_wpr_test_command(recipe_path, test_name) proc = self.start_proc(wpr_test_cmd) old_handler = self.set_sigterm_handler( lambda _signum, _frame: self.handle_sigterm(proc)) if self.xctest_path: parser = xctest_utils.XCTestLogParser() else: parser = gtest_utils.GTestLogParser() test_runner.print_process_output(proc, 'xcodebuild', parser) proc.wait() self.set_sigterm_handler(old_handler) sys.stdout.flush() self.wprgo_stop() return parser, proc.returncode
def testRunTestCaseFail(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_RUN_TEST_CASE_FAIL.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertEqual([], parser.RunningTests()) self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'], parser.FailedTests()) self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled: '], parser.FailureDescription( 'SUIDSandboxUITest.testSUIDSandboxEnabled')) self.assertEqual( ['FAILURE'], parser.TriesForTest('SUIDSandboxUITest.testSUIDSandboxEnabled'))
def testGtestLogParserSkipped(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_SKIPPED.splitlines(): parser.ProcessLine(line) self.assertEqual([], parser.ParsingErrors()) self.assertEqual([], parser.RunningTests()) self.assertEqual([], parser.FailedTests()) self.assertEqual(['ProcessReaderLinux.AbortMessage'], parser.SkippedTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(0, parser.FlakyTests()) self.assertEqual( ['SKIPPED'], parser.TriesForTest('ProcessReaderLinux.AbortMessage'))
def testRunTestCaseParseSwarm(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_SWARM_TEST_FAIL.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertEqual([], parser.RunningTests()) self.assertEqual(['PickleTest.EncodeDecode'], parser.FailedTests()) self.assertEqual([ 'PickleTest.EncodeDecode: ', '../../base/pickle_unittest.cc:69: Failure', 'Value of: false', ' Actual: false', 'Expected: true', ], parser.FailureDescription('PickleTest.EncodeDecode')) self.assertEqual(['FAILURE'], parser.TriesForTest('PickleTest.EncodeDecode'))
def testGTestLogParserMixedStdout(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_MIXED_STDOUT.splitlines(): parser.ProcessLine(line) self.assertEqual([], parser.ParsingErrors()) self.assertEqual(['Crash.Test'], parser.RunningTests()) self.assertEqual(['TestFix.TestCase', 'Crash.Test'], parser.FailedTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(0, parser.FlakyTests()) self.assertEqual(['UNKNOWN'], parser.TriesForTest('Crash.Test')) self.assertEqual(['TIMEOUT'], parser.TriesForTest('TestFix.TestCase')) self.assertEqual( ['SUCCESS'], parser.TriesForTest( 'WebSocketHandshakeHandlerSpdy3Test.RequestResponse'))
def _run(self, cmd, shards=1): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. Returns: GTestResult instance. """ result = gtest_utils.GTestResult(cmd) if self.xctest_path: parser = xctest_utils.XCTestLogParser() else: parser = gtest_utils.GTestLogParser() if shards > 1: test_shards = shard_xctest( os.path.join(self.app_path, self.app_name), shards, self.test_cases) thread_pool = pool.ThreadPool(processes=shards) for out, name, ret in thread_pool.imap_unordered( self.run_tests, test_shards): print "Simulator %s" % name for line in out: print line parser.ProcessLine(line) returncode = ret if ret else 0 thread_pool.close() thread_pool.join() else: # TODO(crbug.com/812705): Implement test sharding for unit tests. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well. proc = subprocess.Popen( cmd, env=self.get_launch_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) while True: line = proc.stdout.readline() if not line: break line = line.rstrip() parser.ProcessLine(line) print line sys.stdout.flush() proc.wait() sys.stdout.flush() returncode = proc.returncode for test in parser.FailedTests(include_flaky=True): # Test cases are named as <test group>.<test case>. If the test case # is prefixed with "FLAKY_", it should be reported as flaked not failed. if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[test] = parser.FailureDescription(test) else: result.failed_tests[test] = parser.FailureDescription(test) result.passed_tests.extend(parser.PassedTests(include_flaky=True)) print '%s returned %s' % (cmd[0], returncode) print # iossim can return 5 if it exits noncleanly even if all tests passed. # Therefore we cannot rely on process exit code to determine success. result.finalize(returncode, parser.CompletedWithoutFailure()) return result
def testNestedGtests(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_NESTED_RUNS.splitlines(): parser.ProcessLine(line) self.assertEqual(['Foo.Bar'], parser.FailedTests(True, True))
def testGTestLogParserSharing(self): # Same tests for log parsing with sharding_supervisor. parser = gtest_utils.GTestLogParser() test_data_shard = TEST_DATA_SHARD_0 + TEST_DATA_SHARD_1 for line in test_data_shard.splitlines(): parser.ProcessLine(line) parser.ProcessLine(TEST_DATA_SHARD_EXIT + '2') self.assertEqual(0, len(parser.ParsingErrors())) self.assertFalse(parser.RunningTests()) self.assertEqual(sorted(FAILURES), sorted(parser.FailedTests())) self.assertEqual(sorted(FAILURES + FAILS_FAILURES), sorted(parser.FailedTests(include_fails=True))) self.assertEqual(sorted(FAILURES + FLAKY_FAILURES), sorted(parser.FailedTests(include_flaky=True))) self.assertEqual( sorted(FAILURES + FAILS_FAILURES + FLAKY_FAILURES), sorted(parser.FailedTests(include_fails=True, include_flaky=True))) self.assertEqual(10, parser.DisabledTests()) self.assertEqual(2, parser.FlakyTests()) test_name = 'NavigationControllerTest.Reload' self.assertEqual('\n'.join(['%s: ' % test_name, RELOAD_ERRORS]), '\n'.join(parser.FailureDescription(test_name))) self.assertEqual(['FAILURE'], parser.TriesForTest(test_name)) test_name = ( 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0') self.assertEqual('\n'.join(['%s: ' % test_name, SPDY_ERRORS]), '\n'.join(parser.FailureDescription(test_name))) self.assertEqual(['FAILURE'], parser.TriesForTest(test_name)) test_name = 'SomeOtherTest.SwitchTypes' self.assertEqual('\n'.join(['%s: ' % test_name, SWITCH_ERRORS]), '\n'.join(parser.FailureDescription(test_name))) self.assertEqual(['FAILURE'], parser.TriesForTest(test_name)) test_name = 'BadTest.TimesOut' self.assertEqual( '\n'.join(['%s: ' % test_name, TIMEOUT_ERRORS, TIMEOUT_MESSAGE]), '\n'.join(parser.FailureDescription(test_name))) self.assertEqual(['TIMEOUT'], parser.TriesForTest(test_name)) test_name = 'MoreBadTest.TimesOutAndFails' self.assertEqual( '\n'.join(['%s: ' % test_name, MOREBAD_ERRORS, TIMEOUT_MESSAGE]), '\n'.join(parser.FailureDescription(test_name))) self.assertEqual(['TIMEOUT'], parser.TriesForTest(test_name)) self.assertEqual(['SUCCESS'], parser.TriesForTest('SomeOtherTest.Foo')) parser = gtest_utils.GTestLogParser() for line in TEST_DATA_CRASH.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertTrue(parser.RunningTests()) self.assertEqual(['HunspellTest.Crashes'], parser.FailedTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(0, parser.FlakyTests()) test_name = 'HunspellTest.Crashes' self.assertEqual('\n'.join(['%s: ' % test_name, 'Did not complete.']), '\n'.join(parser.FailureDescription(test_name))) self.assertEqual(['UNKNOWN'], parser.TriesForTest(test_name))