def testGTestLogParserSharing(self): # Same tests for log parsing with sharding_supervisor. parser = gtest_utils.GTestLogParser() test_data_shard = TEST_DATA_SHARD_0 + TEST_DATA_SHARD_1 for line in test_data_shard.splitlines(): parser.ProcessLine(line) parser.ProcessLine(TEST_DATA_SHARD_EXIT + '2') self.assertEqual(0, len(parser.ParsingErrors())) self.assertFalse(parser.RunningTests()) self.assertEqual(sorted(FAILURES), sorted(parser.FailedTests())) self.assertEqual(sorted(FAILURES + FAILS_FAILURES), sorted(parser.FailedTests(include_fails=True))) self.assertEqual(sorted(FAILURES + FLAKY_FAILURES), sorted(parser.FailedTests(include_flaky=True))) self.assertEqual( sorted(FAILURES + FAILS_FAILURES + FLAKY_FAILURES), sorted(parser.FailedTests(include_fails=True, include_flaky=True))) self.assertEqual(10, parser.DisabledTests()) self.assertEqual(2, parser.FlakyTests()) test_name = 'NavigationControllerTest.Reload' self.assertEqual('\n'.join(['%s: ' % test_name, RELOAD_ERRORS]), '\n'.join(parser.FailureDescription(test_name))) test_name = ( 'NavigationControllerTest/SpdyNetworkTransTest.Constructor/0') self.assertEqual('\n'.join(['%s: ' % test_name, SPDY_ERRORS]), '\n'.join(parser.FailureDescription(test_name))) test_name = 'SomeOtherTest.SwitchTypes' self.assertEqual('\n'.join(['%s: ' % test_name, SWITCH_ERRORS]), '\n'.join(parser.FailureDescription(test_name))) test_name = 'BadTest.TimesOut' self.assertEqual( '\n'.join(['%s: ' % test_name, TIMEOUT_ERRORS, TIMEOUT_MESSAGE]), '\n'.join(parser.FailureDescription(test_name))) test_name = 'MoreBadTest.TimesOutAndFails' self.assertEqual( '\n'.join(['%s: ' % test_name, MOREBAD_ERRORS, TIMEOUT_MESSAGE]), '\n'.join(parser.FailureDescription(test_name))) parser = gtest_utils.GTestLogParser() for line in TEST_DATA_CRASH.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertTrue(parser.RunningTests()) self.assertEqual(['HunspellTest.Crashes'], parser.FailedTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(0, parser.FlakyTests()) test_name = 'HunspellTest.Crashes' self.assertEqual('\n'.join(['%s: ' % test_name, 'Did not complete.']), '\n'.join(parser.FailureDescription(test_name)))
def v0_4(client, options, test_name): """Handles swarm_client/swarming.py starting b39e8cf08c.""" swarming = os.path.join(client, 'swarming.py') cmd = [ sys.executable, swarming, 'collect', '--swarming', options.swarming, '--decorate', test_name, ] print('Running: %s' % ' '.join(cmd)) sys.stdout.flush() proc = subprocess2.Popen(cmd, bufsize=0, stdout=subprocess2.PIPE) gtest_parser = gtest_utils.GTestLogParser() for line in proc.stdout.readlines(): line = line.rstrip() print line gtest_parser.ProcessLine(line) proc.wait() annotation_utils.annotate(test_name, proc.returncode, gtest_parser) print('') return proc.returncode
def testGTestLogParserValgrind(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_VALGRIND.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertFalse(parser.RunningTests()) self.assertFalse(parser.FailedTests()) self.assertEqual([VALGRIND_HASH], parser.SuppressionHashes()) self.assertEqual(VALGRIND_SUPPRESSION, '\n'.join(parser.Suppression(VALGRIND_HASH))) parser = gtest_utils.GTestLogParser() for line in FAILING_TESTS_OUTPUT.splitlines(): parser.ProcessLine(line) self.assertEqual(FAILING_TESTS_EXPECTED, parser.FailedTests(True, True))
def testGTestLogParserValgrind(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_VALGRIND.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertFalse(parser.RunningTests()) self.assertFalse(parser.FailedTests()) self.assertEqual([VALGRIND_HASH], parser.MemoryToolReportHashes()) self.assertEqual(VALGRIND_REPORT, '\n'.join(parser.MemoryToolReport(VALGRIND_HASH))) self.assertEqual(['SUCCESS'], parser.TriesForTest('HunspellTest.All')) parser = gtest_utils.GTestLogParser() for line in FAILING_TESTS_OUTPUT.splitlines(): parser.ProcessLine(line) self.assertEqual(FAILING_TESTS_EXPECTED, parser.FailedTests(True, True))
def __init__(self): buildstep.LogLineObserver.__init__(self) self.gtest_parser = gtest_utils.GTestLogParser() self._master_name_re = re.compile('\[Running for master: "([^"]*)"') self.master_name = '' # Some of our log lines are now big (200K). We need to do this # or twisted will drop the connection and we'll misprocess the log. self.setMaxLineLength(1024*1024)
def testGTestLogParserMixedStdout(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_MIXED_STDOUT.splitlines(): parser.ProcessLine(line) self.assertEqual([], parser.ParsingErrors()) self.assertEqual(['Crash.Test'], parser.RunningTests()) self.assertEqual(['TestFix.TestCase', 'Crash.Test'], parser.FailedTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(0, parser.FlakyTests())
def testRunTestCaseFail(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_RUN_TEST_CASE_FAIL.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertEqual([], parser.RunningTests()) self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'], parser.FailedTests()) self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled: '], parser.FailureDescription( 'SUIDSandboxUITest.testSUIDSandboxEnabled'))
def _Run(command): """Runs the specified command, parsing GTest output. Args: command: The shell command to execute, as a list of arguments. Returns: A GTestResult instance. """ result = utils.GTestResult(command) print ' '.join(command) print 'cwd:', os.getcwd() sys.stdout.flush() proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) parser = gtest_utils.GTestLogParser() while True: line = proc.stdout.readline() if not line: break line = line.rstrip() parser.ProcessLine(line) print line sys.stdout.flush() proc.wait() for test in parser.FailedTests(include_flaky=True): # Tests are named as TestCase.TestName. # A TestName starting with FLAKY_ should not turn the build red. if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[test] = parser.FailureDescription(test) else: result.failed_tests[test] = parser.FailureDescription(test) result.passed_tests.extend(parser.PassedTests(include_flaky=True)) print command[0], 'returned', proc.returncode print sys.stdout.flush() # iossim can return 5 if it exits noncleanly, even if no tests failed. # Therefore we can't rely on this exit code to determine success or failure. result.finalize(proc.returncode, parser.CompletedWithoutFailure()) return result
def testRunTestCaseParseSwarm(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_SWARM_TEST_FAIL.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertEqual([], parser.RunningTests()) self.assertEqual(['PickleTest.EncodeDecode'], parser.FailedTests()) self.assertEqual([ 'PickleTest.EncodeDecode: ', '../../base/pickle_unittest.cc:69: Failure', 'Value of: false', ' Actual: false', 'Expected: true', ], parser.FailureDescription('PickleTest.EncodeDecode'))
def testRunTestCaseTimeout(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_RUN_TEST_CASE_TIMEOUT.splitlines(): parser.ProcessLine(line) self.assertEqual(0, len(parser.ParsingErrors())) self.assertEqual([], parser.RunningTests()) self.assertEqual(['SUIDSandboxUITest.testSUIDSandboxEnabled'], parser.FailedTests()) self.assertEqual( ['SUIDSandboxUITest.testSUIDSandboxEnabled: ', '(junk)'], parser.FailureDescription( 'SUIDSandboxUITest.testSUIDSandboxEnabled')) self.assertEqual( ['TIMEOUT'], parser.TriesForTest('SUIDSandboxUITest.testSUIDSandboxEnabled'))
def testGTestLogParserMixedStdout(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_MIXED_STDOUT.splitlines(): parser.ProcessLine(line) self.assertEqual([], parser.ParsingErrors()) self.assertEqual(['Crash.Test'], parser.RunningTests()) self.assertEqual(['TestFix.TestCase', 'Crash.Test'], parser.FailedTests()) self.assertEqual(0, parser.DisabledTests()) self.assertEqual(0, parser.FlakyTests()) self.assertEqual(['UNKNOWN'], parser.TriesForTest('Crash.Test')) self.assertEqual(['TIMEOUT'], parser.TriesForTest('TestFix.TestCase')) self.assertEqual( ['SUCCESS'], parser.TriesForTest( 'WebSocketHandshakeHandlerSpdy3Test.RequestResponse'))
def v0(client, options, test_name): """This code supports all the earliest versions of swarm_client. This is before --version was added. """ sys.path.insert(0, client) import swarm_get_results # pylint: disable=F0401 timeout = swarm_get_results.DEFAULT_SHARD_WAIT_TIME test_keys = swarm_get_results.get_test_keys(options.swarming, test_name, timeout) if not test_keys: print >> sys.stderr, 'No test keys to get results with.' return 1 if options.shards == -1: options.shards = len(test_keys) elif len(test_keys) < options.shards: print >> sys.stderr, ( 'Warning: Test should have %d shards, but only %d ' 'test keys were found' % (options.shards, len(test_keys))) gtest_parser = gtest_utils.GTestLogParser() exit_code = None shards_remaining = range(len(test_keys)) first_result = True for index, result in swarm_get_results.yield_results( options.swarming, test_keys, timeout, None): assert index == result['config_instance_index'] if first_result and result['num_config_instances'] != len(test_keys): # There are more test_keys than actual shards. shards_remaining = shards_remaining[: result['num_config_instances']] shards_remaining.remove(index) first_result = False output, test_exit_code = gen_shard_output(result, gtest_parser) print output exit_code = max(exit_code, test_exit_code) # Print the annotation before the summary so it's easier to find when scolling # down. annotation_utils.annotate(test_name, exit_code, gtest_parser) print('') return exit_code
def GetSwarmResults( swarm_get_results, swarm_base_url, test_keys, timeout, max_threads): gtest_parser = gtest_utils.GTestLogParser() exit_code = None shards_remaining = range(len(test_keys)) first_result = True for index, result in swarm_get_results.yield_results( swarm_base_url, test_keys, timeout, max_threads): assert index == result['config_instance_index'] if first_result and result['num_config_instances'] != len(test_keys): # There are more test_keys than actual shards. shards_remaining = shards_remaining[:result['num_config_instances']] shards_remaining.remove(index) first_result = False output, test_exit_code = gen_shard_output(result, gtest_parser) print output exit_code = max(exit_code, test_exit_code) output, exit_code = gen_summary_output( gtest_parser.FailedTests(), exit_code, shards_remaining) print output return exit_code
def testNestedGtests(self): parser = gtest_utils.GTestLogParser() for line in TEST_DATA_NESTED_RUNS.splitlines(): parser.ProcessLine(line) self.assertEqual(['Foo.Bar'], parser.FailedTests(True, True))
def GetSwarmResults(swarm_base_url, test_keys): if not test_keys: print 'Error: No test keys to get results with' return 1 gtest_parser = gtest_utils.GTestLogParser() hostnames = ['unknown'] * len(test_keys) exit_codes = [1] * len(test_keys) shard_watcher = ShardWatcher(len(test_keys)) for index in range(len(test_keys)): result_url = '%s/get_result?r=%s' % (swarm_base_url.rstrip('/'), test_keys[index]) while True: output = None try: output = urllib2.urlopen(result_url).read() except urllib2.HTTPError, e: print 'Calling %s threw %s' % (result_url, e) break try: test_outputs = json.loads(output) except (ValueError, TypeError), e: print 'Unable to get results for shard %d' % index print e break if test_outputs['output']: if test_outputs['exit_codes']: test_exit_codes = test_outputs['exit_codes'].split(',') exit_codes[index] = max(map(int, test_exit_codes)) hostnames[index] = test_outputs['hostname'] print print '================================================================' print 'Begin output from shard index %s (%s)' % ( index, hostnames[index]) print '================================================================' print cleaned_output = TestRunOutput(test_outputs['output']) if cleaned_output: for line in cleaned_output.splitlines(): gtest_parser.ProcessLine(line) shard_watcher.ProcessLine(line) print cleaned_output else: # We failed to get any test output which is an error, so we should # show the swarm output since that is probably where the error is. print 'No output produced by the test, it may have failed to run.' print 'Showing all the output, including swarm specific output.' print print test_outputs['output'] # Ensure that we mark this as a failure, since we should always have # output from the tests. if not exit_codes[index]: exit_codes[index] = 1 print '================================================================' print 'End output from shard index %s (%s). Return %d' % ( index, hostnames[index], exit_codes[index]) print '================================================================' print if exit_codes[index] == 0: # If the test passed, delete the key since it is no longer needed. remove_key_url = '%s/cleanup_results' % ( swarm_base_url.rstrip('/')) key_encoding = urllib.urlencode([('r', test_keys[index])]) urllib2.urlopen(remove_key_url, key_encoding) break else: # Test is not yet done, wait a bit before checking again. time.sleep(0.5)