def testGTestLogParserSharing(self):
# Same tests for log parsing with sharding_supervisor.
parser = gtest_utils.GTestLogParser()
test_data_shard = TEST_DATA_SHARD_0 + TEST_DATA_SHARD_1
for line in test_data_shard.splitlines():
parser.ProcessLine(line)
parser.ProcessLine(TEST_DATA_SHARD_EXIT + '2')
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertFalse(parser.RunningTests())
self.assertEqual(sorted(FAILURES), sorted(parser.FailedTests()))
self.assertEqual(sorted(FAILURES + FAILS_FAILURES),
sorted(parser.FailedTests(include_fails=True)))
self.assertEqual(sorted(FAILURES + FLAKY_FAILURES),
sorted(parser.FailedTests(include_flaky=True)))
self.assertEqual(
sorted(FAILURES + FAILS_FAILURES + FLAKY_FAILURES),
sorted(parser.FailedTests(include_fails=True, include_flaky=True)))
self.assertEqual(10, parser.DisabledTests())
self.assertEqual(2, parser.FlakyTests())
test_name = 'NavigationControllerTest.Reload'
self.assertEqual('\n'.join(['%s: ' % test_name, RELOAD_ERRORS]),
'\n'.join(parser.FailureDescription(test_name)))
test_name = (
'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0')
self.assertEqual('\n'.join(['%s: ' % test_name, SPDY_ERRORS]),
'\n'.join(parser.FailureDescription(test_name)))
test_name = 'SomeOtherTest.SwitchTypes'
self.assertEqual('\n'.join(['%s: ' % test_name, SWITCH_ERRORS]),
'\n'.join(parser.FailureDescription(test_name)))
test_name = 'BadTest.TimesOut'
self.assertEqual(
'\n'.join(['%s: ' % test_name, TIMEOUT_ERRORS, TIMEOUT_MESSAGE]),
'\n'.join(parser.FailureDescription(test_name)))
test_name = 'MoreBadTest.TimesOutAndFails'
self.assertEqual(
'\n'.join(['%s: ' % test_name, MOREBAD_ERRORS, TIMEOUT_MESSAGE]),
'\n'.join(parser.FailureDescription(test_name)))
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_CRASH.splitlines():
parser.ProcessLine(line)
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertTrue(parser.RunningTests())
self.assertEqual(['HunspellTest.Crashes'], parser.FailedTests())
self.assertEqual(0, parser.DisabledTests())
self.assertEqual(0, parser.FlakyTests())
test_name = 'HunspellTest.Crashes'
self.assertEqual('\n'.join(['%s: ' % test_name, 'Did not complete.']),
'\n'.join(parser.FailureDescription(test_name)))
def v0_4(client, options, test_name):
"""Handles swarm_client/swarming.py starting b39e8cf08c."""
swarming = os.path.join(client, 'swarming.py')
cmd = [
sys.executable,
swarming,
'collect',
'--swarming',
options.swarming,
'--decorate',
test_name,
]
print('Running: %s' % ' '.join(cmd))
sys.stdout.flush()
proc = subprocess2.Popen(cmd, bufsize=0, stdout=subprocess2.PIPE)
gtest_parser = gtest_utils.GTestLogParser()
for line in proc.stdout.readlines():
line = line.rstrip()
print line
gtest_parser.ProcessLine(line)
proc.wait()
annotation_utils.annotate(test_name, proc.returncode, gtest_parser)
print('')
return proc.returncode
def testGTestLogParserValgrind(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_VALGRIND.splitlines():
parser.ProcessLine(line)
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertFalse(parser.RunningTests())
self.assertFalse(parser.FailedTests())
self.assertEqual([VALGRIND_HASH], parser.SuppressionHashes())
self.assertEqual(VALGRIND_SUPPRESSION,
'\n'.join(parser.Suppression(VALGRIND_HASH)))
parser = gtest_utils.GTestLogParser()
for line in FAILING_TESTS_OUTPUT.splitlines():
parser.ProcessLine(line)
self.assertEqual(FAILING_TESTS_EXPECTED,
parser.FailedTests(True, True))
def testGTestLogParserValgrind(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_VALGRIND.splitlines():
parser.ProcessLine(line)
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertFalse(parser.RunningTests())
self.assertFalse(parser.FailedTests())
self.assertEqual([VALGRIND_HASH], parser.MemoryToolReportHashes())
self.assertEqual(VALGRIND_REPORT,
'\n'.join(parser.MemoryToolReport(VALGRIND_HASH)))
self.assertEqual(['SUCCESS'], parser.TriesForTest('HunspellTest.All'))
parser = gtest_utils.GTestLogParser()
for line in FAILING_TESTS_OUTPUT.splitlines():
parser.ProcessLine(line)
self.assertEqual(FAILING_TESTS_EXPECTED,
parser.FailedTests(True, True))
def __init__(self):
buildstep.LogLineObserver.__init__(self)
self.gtest_parser = gtest_utils.GTestLogParser()
self._master_name_re = re.compile('\[Running for master: "([^"]*)"')
self.master_name = ''
# Some of our log lines are now big (200K). We need to do this
# or twisted will drop the connection and we'll misprocess the log.
self.setMaxLineLength(1024*1024)
def testGTestLogParserMixedStdout(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_MIXED_STDOUT.splitlines():
parser.ProcessLine(line)
self.assertEqual([], parser.ParsingErrors())
self.assertEqual(['Crash.Test'], parser.RunningTests())
self.assertEqual(['TestFix.TestCase', 'Crash.Test'],
parser.FailedTests())
self.assertEqual(0, parser.DisabledTests())
self.assertEqual(0, parser.FlakyTests())
def testRunTestCaseFail(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_RUN_TEST_CASE_FAIL.splitlines():
parser.ProcessLine(line)
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertEqual([], parser.RunningTests())
self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'],
parser.FailedTests())
self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled: '],
parser.FailureDescription(
'SUIDSandboxUITest.testSUIDSandboxEnabled'))
def _Run(command):
"""Runs the specified command, parsing GTest output.
Args:
command: The shell command to execute, as a list of arguments.
Returns:
A GTestResult instance.
"""
result = utils.GTestResult(command)
print ' '.join(command)
print 'cwd:', os.getcwd()
sys.stdout.flush()
proc = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
parser = gtest_utils.GTestLogParser()
while True:
line = proc.stdout.readline()
if not line:
break
line = line.rstrip()
parser.ProcessLine(line)
print line
sys.stdout.flush()
proc.wait()
for test in parser.FailedTests(include_flaky=True):
# Tests are named as TestCase.TestName.
# A TestName starting with FLAKY_ should not turn the build red.
if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'):
result.flaked_tests[test] = parser.FailureDescription(test)
else:
result.failed_tests[test] = parser.FailureDescription(test)
result.passed_tests.extend(parser.PassedTests(include_flaky=True))
print command[0], 'returned', proc.returncode
print
sys.stdout.flush()
# iossim can return 5 if it exits noncleanly, even if no tests failed.
# Therefore we can't rely on this exit code to determine success or failure.
result.finalize(proc.returncode, parser.CompletedWithoutFailure())
return result
def testRunTestCaseParseSwarm(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_SWARM_TEST_FAIL.splitlines():
parser.ProcessLine(line)
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertEqual([], parser.RunningTests())
self.assertEqual(['PickleTest.EncodeDecode'], parser.FailedTests())
self.assertEqual([
'PickleTest.EncodeDecode: ',
'../../base/pickle_unittest.cc:69: Failure',
'Value of: false',
' Actual: false',
'Expected: true',
], parser.FailureDescription('PickleTest.EncodeDecode'))
def testRunTestCaseTimeout(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_RUN_TEST_CASE_TIMEOUT.splitlines():
parser.ProcessLine(line)
self.assertEqual(0, len(parser.ParsingErrors()))
self.assertEqual([], parser.RunningTests())
self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'],
parser.FailedTests())
self.assertEqual(
['SUIDSandboxUITest.testSUIDSandboxEnabled: ', '(junk)'],
parser.FailureDescription(
'SUIDSandboxUITest.testSUIDSandboxEnabled'))
self.assertEqual(
['TIMEOUT'],
parser.TriesForTest('SUIDSandboxUITest.testSUIDSandboxEnabled'))
def testGTestLogParserMixedStdout(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_MIXED_STDOUT.splitlines():
parser.ProcessLine(line)
self.assertEqual([], parser.ParsingErrors())
self.assertEqual(['Crash.Test'], parser.RunningTests())
self.assertEqual(['TestFix.TestCase', 'Crash.Test'],
parser.FailedTests())
self.assertEqual(0, parser.DisabledTests())
self.assertEqual(0, parser.FlakyTests())
self.assertEqual(['UNKNOWN'], parser.TriesForTest('Crash.Test'))
self.assertEqual(['TIMEOUT'], parser.TriesForTest('TestFix.TestCase'))
self.assertEqual(
['SUCCESS'],
parser.TriesForTest(
'WebSocketHandshakeHandlerSpdy3Test.RequestResponse'))
def v0(client, options, test_name):
"""This code supports all the earliest versions of swarm_client.
This is before --version was added.
"""
sys.path.insert(0, client)
import swarm_get_results # pylint: disable=F0401
timeout = swarm_get_results.DEFAULT_SHARD_WAIT_TIME
test_keys = swarm_get_results.get_test_keys(options.swarming, test_name,
timeout)
if not test_keys:
print >> sys.stderr, 'No test keys to get results with.'
return 1
if options.shards == -1:
options.shards = len(test_keys)
elif len(test_keys) < options.shards:
print >> sys.stderr, (
'Warning: Test should have %d shards, but only %d '
'test keys were found' % (options.shards, len(test_keys)))
gtest_parser = gtest_utils.GTestLogParser()
exit_code = None
shards_remaining = range(len(test_keys))
first_result = True
for index, result in swarm_get_results.yield_results(
options.swarming, test_keys, timeout, None):
assert index == result['config_instance_index']
if first_result and result['num_config_instances'] != len(test_keys):
# There are more test_keys than actual shards.
shards_remaining = shards_remaining[:
result['num_config_instances']]
shards_remaining.remove(index)
first_result = False
output, test_exit_code = gen_shard_output(result, gtest_parser)
print output
exit_code = max(exit_code, test_exit_code)
# Print the annotation before the summary so it's easier to find when scolling
# down.
annotation_utils.annotate(test_name, exit_code, gtest_parser)
print('')
return exit_code
def GetSwarmResults(
swarm_get_results, swarm_base_url, test_keys, timeout, max_threads):
gtest_parser = gtest_utils.GTestLogParser()
exit_code = None
shards_remaining = range(len(test_keys))
first_result = True
for index, result in swarm_get_results.yield_results(
swarm_base_url, test_keys, timeout, max_threads):
assert index == result['config_instance_index']
if first_result and result['num_config_instances'] != len(test_keys):
# There are more test_keys than actual shards.
shards_remaining = shards_remaining[:result['num_config_instances']]
shards_remaining.remove(index)
first_result = False
output, test_exit_code = gen_shard_output(result, gtest_parser)
print output
exit_code = max(exit_code, test_exit_code)
output, exit_code = gen_summary_output(
gtest_parser.FailedTests(),
exit_code,
shards_remaining)
print output
return exit_code
def testNestedGtests(self):
parser = gtest_utils.GTestLogParser()
for line in TEST_DATA_NESTED_RUNS.splitlines():
parser.ProcessLine(line)
self.assertEqual(['Foo.Bar'], parser.FailedTests(True, True))
def GetSwarmResults(swarm_base_url, test_keys):
if not test_keys:
print 'Error: No test keys to get results with'
return 1
gtest_parser = gtest_utils.GTestLogParser()
hostnames = ['unknown'] * len(test_keys)
exit_codes = [1] * len(test_keys)
shard_watcher = ShardWatcher(len(test_keys))
for index in range(len(test_keys)):
result_url = '%s/get_result?r=%s' % (swarm_base_url.rstrip('/'),
test_keys[index])
while True:
output = None
try:
output = urllib2.urlopen(result_url).read()
except urllib2.HTTPError, e:
print 'Calling %s threw %s' % (result_url, e)
break
try:
test_outputs = json.loads(output)
except (ValueError, TypeError), e:
print 'Unable to get results for shard %d' % index
print e
break
if test_outputs['output']:
if test_outputs['exit_codes']:
test_exit_codes = test_outputs['exit_codes'].split(',')
exit_codes[index] = max(map(int, test_exit_codes))
hostnames[index] = test_outputs['hostname']
print
print '================================================================'
print 'Begin output from shard index %s (%s)' % (
index, hostnames[index])
print '================================================================'
print
cleaned_output = TestRunOutput(test_outputs['output'])
if cleaned_output:
for line in cleaned_output.splitlines():
gtest_parser.ProcessLine(line)
shard_watcher.ProcessLine(line)
print cleaned_output
else:
# We failed to get any test output which is an error, so we should
# show the swarm output since that is probably where the error is.
print 'No output produced by the test, it may have failed to run.'
print 'Showing all the output, including swarm specific output.'
print
print test_outputs['output']
# Ensure that we mark this as a failure, since we should always have
# output from the tests.
if not exit_codes[index]:
exit_codes[index] = 1
print '================================================================'
print 'End output from shard index %s (%s). Return %d' % (
index, hostnames[index], exit_codes[index])
print '================================================================'
print
if exit_codes[index] == 0:
# If the test passed, delete the key since it is no longer needed.
remove_key_url = '%s/cleanup_results' % (
swarm_base_url.rstrip('/'))
key_encoding = urllib.urlencode([('r', test_keys[index])])
urllib2.urlopen(remove_key_url, key_encoding)
break
else:
# Test is not yet done, wait a bit before checking again.
time.sleep(0.5)
