def testXCTestLogParserStdout(self):
parser = xctest_utils.XCTestLogParser()
for line in TEST_SUCCEEDED_DATA.splitlines():
parser.ProcessLine(line)
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertFalse(parser.RunningTests())
self.assertFalse(parser.FailedTests())
self.assertEqual(['webShellTest.testOne'], parser.PassedTests())
self.assertEqual(['SUCCESS'], parser.TriesForTest('webShellTest.testOne'))
self.assertTrue(parser.CompletedWithoutFailure())
parser = xctest_utils.XCTestLogParser()
for line in TEST_MIXED_DATA.splitlines():
parser.ProcessLine(line)
self.assertEqual(sorted(FAILING_TESTS_EXPECTED),
parser.FailedTests(True, True))
test_name = 'ChromeSmokeTestCase.testTapToolsMenu'
self.assertEqual('\n'.join(['%s: ' % test_name, TAP_ERRORS]),
'\n'.join(parser.FailureDescription(test_name)))
self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
test_name = 'ChromeSmokeTestCase.testReload'
self.assertEqual('\n'.join(['%s: ' % test_name, RELOAD_ERRORS]),
'\n'.join(parser.FailureDescription(test_name)))
self.assertEqual(['FAILURE'], parser.TriesForTest(test_name))
def _run(self, cmd, shards=1):
"""Runs the specified command, parsing GTest output.
Args:
cmd: List of strings forming the command to run.
Returns:
GTestResult instance.
"""
result = gtest_utils.GTestResult(cmd)
if self.webrtc_xctest:
parser = xctest_utils.XCTestLogParser()
else:
parser = gtest_utils.GTestLogParser()
# TODO(crbug.com/812705): Implement test sharding for unit tests.
# TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
proc = self.start_proc(cmd)
old_handler = self.set_sigterm_handler(
lambda _signum, _frame: self.handle_sigterm(proc))
print_process_output(proc, 'xcodebuild', parser)
LOGGER.info('Waiting for test process to terminate.')
proc.wait()
LOGGER.info('Test process terminated.')
self.set_sigterm_handler(old_handler)
sys.stdout.flush()
LOGGER.debug('Stdout flushed after test process.')
returncode = proc.returncode
if self.webrtc_xctest and parser.SystemAlertPresent():
raise SystemAlertPresentError()
LOGGER.debug('Processing test results.')
for test in parser.FailedTests(include_flaky=True):
# Test cases are named as <test group>.<test case>. If the test case
# is prefixed with "FLAKY_", it should be reported as flaked not failed.
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[test] = parser.FailureDescription(test)
else:
result.failed_tests[test] = parser.FailureDescription(test)
result.passed_tests.extend(parser.PassedTests(include_flaky=True))
# Only GTest outputs compiled tests in a json file.
if not self.webrtc_xctest:
result.disabled_tests_from_compiled_tests_file.extend(
parser.DisabledTestsFromCompiledTestsFile())
LOGGER.info('%s returned %s\n', cmd[0], returncode)
# xcodebuild can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
result.finalize(returncode, parser.CompletedWithoutFailure())
return result
def _run(self, cmd):
"""Runs the specified command, parsing GTest output.
Args:
cmd: List of strings forming the command to run.
Returns:
GTestResult instance.
"""
print ' '.join(cmd)
print
result = gtest_utils.GTestResult(cmd)
if self.xctest_path:
parser = xctest_utils.XCTestLogParser()
else:
parser = gtest_utils.GTestLogParser()
proc = subprocess.Popen(
cmd,
env=self.get_launch_env(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
while True:
line = proc.stdout.readline()
if not line:
break
line = line.rstrip()
parser.ProcessLine(line)
print line
sys.stdout.flush()
proc.wait()
sys.stdout.flush()
for test in parser.FailedTests(include_flaky=True):
# Test cases are named as <test group>.<test case>. If the test case
# is prefixed with "FLAKY_", it should be reported as flaked not failed.
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[test] = parser.FailureDescription(test)
else:
result.failed_tests[test] = parser.FailureDescription(test)
result.passed_tests.extend(parser.PassedTests(include_flaky=True))
print '%s returned %s' % (cmd[0], proc.returncode)
print
# iossim can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
result.finalize(proc.returncode, parser.CompletedWithoutFailure())
return result
def run_wpr_test(self, udid, test_name, recipe_path, replay_path):
"""Runs a single WPR test.
Args:
udid: UDID for the simulator to run the test on
test_name: Test name(format: ios_website) of this wpr test.
recipe_path: Path to the recipe file (i.e. ios_costco.test)
replay_path: Path to the replay file (i.e. ios_costco)
Returns
[parser, return code from test] where
parser: a XCTest or GTestLogParser which has processed all
the output from the test
"""
LOGGER.info('Running test for recipe %s', recipe_path)
self.wprgo_start(replay_path)
# TODO(crbug.com/881096): Consider reusing get_launch_command
# and adding the autofillautomation flag to it
# TODO(crbug.com/881096): We only run AutofillAutomationTestCase
# as we have other unit tests in the suite which are not related
# to testing website recipe/replays. We should consider moving
# one or the other to a different suite.
# For the website replay test suite, we need to pass in a single
# recipe at a time, with flags "autofillautomation={recipe_path}",
# "--enable-features=AutofillShowTypePredictions". The args are written in
# xctestrun file, which is produced through EgtestsApp and LaunchCommand
# defined in xcodebuild_runner.
wpr_test_cmd = self.get_wpr_test_command(recipe_path, test_name)
proc = self.start_proc(wpr_test_cmd)
old_handler = self.set_sigterm_handler(
lambda _signum, _frame: self.handle_sigterm(proc))
if self.xctest_path:
parser = xctest_utils.XCTestLogParser()
else:
parser = gtest_utils.GTestLogParser()
test_runner.print_process_output(proc, 'xcodebuild', parser)
proc.wait()
self.set_sigterm_handler(old_handler)
sys.stdout.flush()
self.wprgo_stop()
return parser, proc.returncode
def run_tests(self, cmd):
"""Runs passed-in tests. Builds a command and create a simulator to
run tests.
Args:
cmd: A running command.
Return:
out: (list) List of strings of subprocess's output.
returncode: (int) Return code of subprocess.
"""
proc = self.start_proc(cmd)
out = print_process_output(proc, 'xcodebuild',
xctest_utils.XCTestLogParser())
self.deleteSimulator(self.udid)
return (out, proc.returncode)
def _run(self, cmd, shards=1):
"""Runs the specified command, parsing GTest output.
Args:
cmd: List of strings forming the command to run.
Returns:
GTestResult instance.
"""
result = gtest_utils.GTestResult(cmd)
if self.xctest_path:
parser = xctest_utils.XCTestLogParser()
else:
parser = gtest_utils.GTestLogParser()
if shards > 1:
test_shards = shard_xctest(
os.path.join(self.app_path, self.app_name), shards,
self.test_cases)
thread_pool = pool.ThreadPool(processes=shards)
for out, name, ret in thread_pool.imap_unordered(
self.run_tests, test_shards):
print "Simulator %s" % name
for line in out:
print line
parser.ProcessLine(line)
returncode = ret if ret else 0
thread_pool.close()
thread_pool.join()
else:
# TODO(crbug.com/812705): Implement test sharding for unit tests.
# TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
proc = subprocess.Popen(
cmd,
env=self.get_launch_env(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
while True:
line = proc.stdout.readline()
if not line:
break
line = line.rstrip()
parser.ProcessLine(line)
print line
sys.stdout.flush()
proc.wait()
sys.stdout.flush()
returncode = proc.returncode
for test in parser.FailedTests(include_flaky=True):
# Test cases are named as <test group>.<test case>. If the test case
# is prefixed with "FLAKY_", it should be reported as flaked not failed.
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[test] = parser.FailureDescription(test)
else:
result.failed_tests[test] = parser.FailureDescription(test)
result.passed_tests.extend(parser.PassedTests(include_flaky=True))
print '%s returned %s' % (cmd[0], returncode)
print
# iossim can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
result.finalize(returncode, parser.CompletedWithoutFailure())
return result