def _run(self, cmd, shards=1):
"""Runs the specified command, parsing GTest output.
Args:
cmd: List of strings forming the command to run.
Returns:
GTestResult instance.
"""
result = gtest_utils.GTestResult(cmd)
if self.webrtc_xctest:
parser = xctest_utils.XCTestLogParser()
else:
parser = gtest_utils.GTestLogParser()
# TODO(crbug.com/812705): Implement test sharding for unit tests.
# TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
proc = self.start_proc(cmd)
old_handler = self.set_sigterm_handler(
lambda _signum, _frame: self.handle_sigterm(proc))
print_process_output(proc, 'xcodebuild', parser)
LOGGER.info('Waiting for test process to terminate.')
proc.wait()
LOGGER.info('Test process terminated.')
self.set_sigterm_handler(old_handler)
sys.stdout.flush()
LOGGER.debug('Stdout flushed after test process.')
returncode = proc.returncode
if self.webrtc_xctest and parser.SystemAlertPresent():
raise SystemAlertPresentError()
LOGGER.debug('Processing test results.')
for test in parser.FailedTests(include_flaky=True):
# Test cases are named as <test group>.<test case>. If the test case
# is prefixed with "FLAKY_", it should be reported as flaked not failed.
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[test] = parser.FailureDescription(test)
else:
result.failed_tests[test] = parser.FailureDescription(test)
result.passed_tests.extend(parser.PassedTests(include_flaky=True))
# Only GTest outputs compiled tests in a json file.
if not self.webrtc_xctest:
result.disabled_tests_from_compiled_tests_file.extend(
parser.DisabledTestsFromCompiledTestsFile())
LOGGER.info('%s returned %s\n', cmd[0], returncode)
# xcodebuild can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
result.finalize(returncode, parser.CompletedWithoutFailure())
return result
def _run(self, cmd):
"""Runs the specified command, parsing GTest output.
Args:
cmd: List of strings forming the command to run.
Returns:
GTestResult instance.
"""
print ' '.join(cmd)
print
result = gtest_utils.GTestResult(cmd)
if self.xctest_path:
parser = xctest_utils.XCTestLogParser()
else:
parser = gtest_utils.GTestLogParser()
proc = subprocess.Popen(
cmd,
env=self.get_launch_env(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
while True:
line = proc.stdout.readline()
if not line:
break
line = line.rstrip()
parser.ProcessLine(line)
print line
sys.stdout.flush()
proc.wait()
sys.stdout.flush()
for test in parser.FailedTests(include_flaky=True):
# Test cases are named as <test group>.<test case>. If the test case
# is prefixed with "FLAKY_", it should be reported as flaked not failed.
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[test] = parser.FailureDescription(test)
else:
result.failed_tests[test] = parser.FailureDescription(test)
result.passed_tests.extend(parser.PassedTests(include_flaky=True))
print '%s returned %s' % (cmd[0], proc.returncode)
print
# iossim can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
result.finalize(proc.returncode, parser.CompletedWithoutFailure())
return result
def _run(self, cmd, shards=1):
"""Runs the specified command, parsing GTest output.
Args:
cmd: List of strings forming the command to run.
Returns:
GTestResult instance.
"""
result = gtest_utils.GTestResult(cmd)
if self.xctest_path:
parser = xctest_utils.XCTestLogParser()
else:
parser = gtest_utils.GTestLogParser()
if shards > 1:
test_shards = shard_xctest(
os.path.join(self.app_path, self.app_name), shards,
self.test_cases)
thread_pool = pool.ThreadPool(processes=shards)
for out, name, ret in thread_pool.imap_unordered(
self.run_tests, test_shards):
print "Simulator %s" % name
for line in out:
print line
parser.ProcessLine(line)
returncode = ret if ret else 0
thread_pool.close()
thread_pool.join()
else:
# TODO(crbug.com/812705): Implement test sharding for unit tests.
# TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
proc = subprocess.Popen(
cmd,
env=self.get_launch_env(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
while True:
line = proc.stdout.readline()
if not line:
break
line = line.rstrip()
parser.ProcessLine(line)
print line
sys.stdout.flush()
proc.wait()
sys.stdout.flush()
returncode = proc.returncode
for test in parser.FailedTests(include_flaky=True):
# Test cases are named as <test group>.<test case>. If the test case
# is prefixed with "FLAKY_", it should be reported as flaked not failed.
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[test] = parser.FailureDescription(test)
else:
result.failed_tests[test] = parser.FailureDescription(test)
result.passed_tests.extend(parser.PassedTests(include_flaky=True))
print '%s returned %s' % (cmd[0], returncode)
print
# iossim can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
result.finalize(returncode, parser.CompletedWithoutFailure())
return result
def _run(self, cmd, shards=1):
"""Runs the specified command, parsing GTest output.
Args:
cmd: List of strings forming the command to run.
NOTE: in the case of WprProxySimulatorTestRunner, cmd
is a dict forming the configuration for the test (including
filter rules), and not indicative of the actual command
we build and execute in _run.
Returns:
GTestResult instance.
Raises:
ShardingDisabledError: If shards > 1 as currently sharding is not
supported.
SystemAlertPresentError: If system alert is shown on the device.
"""
result = gtest_utils.GTestResult(cmd)
completed_without_failure = True
total_returncode = 0
if shards > 1:
# TODO(crbug.com/881096): reimplement sharding in the future
raise test_runner.ShardingDisabledError()
# TODO(crbug.com/812705): Implement test sharding for unit tests.
# TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well.
# Create a simulator for these tests, and prepare it with the
# certificate needed for HTTPS proxying.
udid = self.getSimulator()
self.copy_trusted_certificate()
for recipe_path in glob.glob('{}/*.test'.format(self.replay_path)):
base_name = os.path.basename(recipe_path)
test_name = os.path.splitext(base_name)[0]
replay_path = '{}/{}'.format(self.replay_path, test_name)
if self.should_run_wpr_test(test_name, cmd['test_filter'], cmd['invert']):
parser, returncode = self.run_wpr_test(udid, test_name, recipe_path,
replay_path)
# If this test fails, immediately rerun it to see if it deflakes.
# We simply overwrite the first result with the second.
if parser.FailedTests(include_flaky=True):
parser, returncode = self.run_wpr_test(udid, test_name, recipe_path,
replay_path)
for test in parser.FailedTests(include_flaky=True):
# All test names will be the same since we re-run the same suite;
# therefore, to differentiate the results, we append the recipe
# name to the test suite.
testWithRecipeName = "{}.{}".format(base_name, test)
# Test cases are named as <test group>.<test case>. If the test case
# is prefixed w/"FLAKY_", it should be reported as flaked not failed
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[testWithRecipeName] = parser.FailureDescription(
test)
else:
result.failed_tests[testWithRecipeName] = parser.FailureDescription(
test)
for test in parser.PassedTests(include_flaky=True):
testWithRecipeName = "{}.{}".format(base_name, test)
result.passed_tests.extend([testWithRecipeName])
# Check for runtime errors.
if self.xctest_path and parser.SystemAlertPresent():
raise test_runner.SystemAlertPresentError()
if returncode != 0:
total_returncode = returncode
if not parser.CompletedWithoutFailure():
completed_without_failure = False
LOGGER.info('%s test returned %s\n', recipe_path, returncode)
self.deleteSimulator(udid)
# xcodebuild can return 5 if it exits noncleanly even if all tests passed.
# Therefore we cannot rely on process exit code to determine success.
# NOTE: total_returncode is 0 OR the last non-zero return code from a test.
result.finalize(total_returncode, completed_without_failure)
return result
