def testGetSignalFromStepLogFlaky(self):
master_name = 'm'
builder_name = 'b'
build_number = 124
step_name = 'abc_test'
failure_info = {
'master_name': 'm',
'builder_name': 'b',
'build_number': 124,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'abc_test': {
'last_pass': 123,
'current_failure': 124,
'first_failure': 124,
}
}
}
self.MockGetStdiolog(master_name, builder_name, build_number,
step_name)
self.MockGetGtestJsonResult()
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
step = WfStep.Get(master_name, builder_name, build_number, step_name)
self.assertIsNotNone(step)
self.assertIsNotNone(step.log_data)
self.assertEqual('flaky', step.log_data)
self.assertEqual({}, signals['abc_test']['files'])
def testGetSignalFromStepLogFlaky(self):
master_name = 'm'
builder_name = 'b'
build_number = 124
step_name = 'abc_test'
failure_info = {
'master_name': 'm',
'builder_name': 'b',
'build_number': 124,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'abc_test': {
'last_pass': 123,
'current_failure': 124,
'first_failure': 124,
}
}
}
self.MockGetStdiolog(master_name, builder_name, build_number, step_name)
self.MockGetGtestJsonResult()
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
step = WfStep.Get(master_name, builder_name, build_number, step_name)
self.assertIsNotNone(step)
self.assertIsNotNone(step.log_data)
self.assertEqual('flaky', step.log_data)
self.assertEqual({}, signals['abc_test']['files'])
def testGetSignalFromStepLog(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
# Mock both stdiolog and gtest json results to test whether Findit will
# go to step log first when both logs exist.
self.MockGetStdiolog(master_name, builder_name, build_number, step_name)
self.MockGetGtestJsonResult()
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
signals = pipeline.run(self.FAILURE_INFO, False)
step = WfStep.Get(master_name, builder_name, build_number, step_name)
expected_files = {
'a/b/u2s1.cc': [567],
'a/b/u3s2.cc': [110]
}
self.assertIsNotNone(step)
self.assertIsNotNone(step.log_data)
self.assertEqual(expected_files, signals['abc_test']['files'])
def testBailOutForUnsupportedStep(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
supported_step_name = 'abc_test'
unsupported_step_name = 'unsupported_step6'
failure_info = {
'master_name': master_name,
'builder_name': 'b',
'build_number': 123,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
supported_step_name: {
'last_pass': 122,
'current_failure': 123,
'first_failure': 123,
},
unsupported_step_name: {
}
}
}
def MockGetGtestResultLog(*_):
return None
self.mock(buildbot, 'GetGtestResultLog', MockGetGtestResultLog)
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(FAILURE_SIGNALS, signals)
def testGetSignalFromStepLog(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
# Mock both stdiolog and gtest json results to test whether Findit will
# go to step log first when both logs exist.
self.MockGetGtestJsonResult()
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline(FAILURE_INFO)
signals = pipeline.run(FAILURE_INFO)
step = WfStep.Get(master_name, builder_name, build_number, step_name)
expected_files = {
'a/b/u2s1.cc': [567],
'a/b/u3s2.cc': [110]
}
self.assertIsNotNone(step)
self.assertIsNotNone(step.log_data)
self.assertEqual(expected_files, signals['abc_test']['files'])
def testBailOutForUnsupportedStep(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
supported_step_name = 'abc_test'
unsupported_step_name = 'unsupported_step6'
failure_info = {
'master_name': master_name,
'builder_name': 'b',
'build_number': 123,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
supported_step_name: {
'last_pass': 122,
'current_failure': 123,
'first_failure': 123,
},
unsupported_step_name: {
}
}
}
def MockGetGtestResultLog(*_):
return None
self.MockGetStdiolog(master_name, builder_name, build_number,
supported_step_name)
self.mock(buildbot, 'GetGtestResultLog', MockGetGtestResultLog)
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info, False)
self.assertEqual(self.FAILURE_SIGNALS, signals)
def testBailOutIfNotAFailedBuild(self):
failure_info = {
'failed': False,
}
expected_signals = {}
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(expected_signals, signals)
def testBailOutIfNotAFailedBuild(self):
failure_info = {
'failed': False,
}
expected_signals = {}
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(expected_signals, signals)
def testBailOutIfNoValidChromiumRevision(self):
failure_info = {
'failed': True,
'chromium_revision': None,
}
expected_signals = {}
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(expected_signals, signals)
def testBailOutIfNoValidChromiumRevision(self):
failure_info = {
'failed': True,
'chromium_revision': None,
}
expected_signals = {}
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(expected_signals, signals)
def testBailOutIfInfraFailure(self):
failure_info = {
'failed': True,
'failure_type': failure_type.INFRA,
'chromium_revision': '00baf00ba',
}
expected_signals = {}
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(expected_signals, signals)
def testExtractSignalsForTestsFlaky(self):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'abc_test': {
'last_pass': 221,
'current_failure': 223,
'first_failure': 222,
'tests': {
'Unittest2.Subtest1': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
},
'Unittest3.Subtest2': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
}
}
}
}
}
step = WfStep.Create(master_name, builder_name, build_number, 'abc_test')
step.isolated = True
step.log_data = 'flaky'
step.put()
expected_signals = {
'abc_test': {
'files': {},
'keywords': {},
'tests': {}
}
}
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(expected_signals, signals)
def testExtractSignalsForTestsFlaky(self):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'abc_test': {
'last_pass': 221,
'current_failure': 223,
'first_failure': 222,
'tests': {
'Unittest2.Subtest1': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
},
'Unittest3.Subtest2': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
}
}
}
}
}
step = WfStep.Create(master_name, builder_name, build_number, 'abc_test')
step.isolated = True
step.log_data = 'flaky'
step.put()
expected_signals = {
'abc_test': {
'files': {},
'keywords': {},
'tests': {}
}
}
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info, False)
self.assertEqual(expected_signals, signals)
def testWfStepStdioLogNotDownloadedYet(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
self.MockGetStdiolog(master_name, builder_name, build_number, step_name)
pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
pipeline.start()
self.execute_queued_tasks()
step = WfStep.Create(master_name, builder_name, build_number, step_name)
self.assertIsNotNone(step)
def testWfStepStdioLogNotDownloadedYet(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline(FAILURE_INFO)
pipeline.start()
self.execute_queued_tasks()
step = WfStep.Create(master_name, builder_name, build_number, step_name)
self.assertIsNotNone(step)
def testWfStepStdioLogAlreadyDownloaded(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
step = WfStep.Create(master_name, builder_name, build_number, step_name)
step.log_data = ABC_TEST_FAILURE_LOG
step.put()
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline(FAILURE_INFO)
signals = pipeline.run(FAILURE_INFO)
self.assertEqual(FAILURE_SIGNALS, signals)
def testExtractStorablePortionOfLogWithBigLogData(self): self.mock(ExtractSignalPipeline, 'LOG_DATA_BYTE_LIMIT', 500) lines = [str(9 - i) * 99 for i in range(9)] log_data = '\n'.join(lines) expected_result = '\n'.join(lines[-5:]) result = ExtractSignalPipeline._ExtractStorablePortionOfLog(log_data) self.assertEqual(expected_result, result)
def run(self, master_name, builder_name, build_number, build_completed,
force):
self._ResetAnalysis(master_name, builder_name, build_number)
# The yield statements below return PipelineFutures, which allow subsequent
# pipelines to refer to previous output values.
# https://github.com/GoogleCloudPlatform/appengine-pipelines/wiki/Python
# Heuristic Approach.
failure_info = yield DetectFirstFailurePipeline(
master_name, builder_name, build_number)
change_logs = yield PullChangelogPipeline(failure_info)
deps_info = yield ExtractDEPSInfoPipeline(failure_info, change_logs)
signals = yield ExtractSignalPipeline(failure_info)
heuristic_result = yield IdentifyCulpritPipeline(
failure_info, change_logs, deps_info, signals, build_completed)
# Try job approach.
with pipeline.InOrder():
# Swarming rerun.
# Triggers swarming tasks when first time test failure happens.
# This pipeline will run before build completes.
yield TriggerSwarmingTasksPipeline(master_name, builder_name,
build_number, failure_info,
force)
# Checks if first time failures happen and starts a try job if yes.
yield StartTryJobOnDemandPipeline(master_name, builder_name,
build_number, failure_info,
signals, heuristic_result,
build_completed, force)
# Trigger flake analysis on flaky tests, if any.
yield TriggerFlakeAnalysesPipeline(master_name, builder_name,
build_number)
def testWfStepStdioLogNotDownloadedYet(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
self.MockGetStdiolog(master_name, builder_name, build_number,
step_name)
pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
pipeline.start()
self.execute_queued_tasks()
step = WfStep.Create(master_name, builder_name, build_number,
step_name)
self.assertIsNotNone(step)
def testExtractStorablePortionOfLogWithBigLogData(self):
self.mock(ExtractSignalPipeline, 'LOG_DATA_BYTE_LIMIT', 500)
lines = [str(9 - i) * 99 for i in range(9)]
log_data = '\n'.join(lines)
expected_result = '\n'.join(lines[-5:])
result = ExtractSignalPipeline._ExtractStorablePortionOfLog(log_data)
self.assertEqual(expected_result, result)
def testWfStepStdioLogAlreadyDownloaded(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
step = WfStep.Create(master_name, builder_name, build_number, step_name)
step.log_data = self.ABC_TEST_FAILURE_LOG
step.put()
step_log_url = buildbot.CreateStdioLogUrl(
master_name, builder_name, build_number, step_name)
with self.mock_urlfetch() as urlfetch:
urlfetch.register_handler(step_log_url, 'If used, test should fail!')
pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
signals = pipeline.run(self.FAILURE_INFO)
self.assertEqual(self.FAILURE_SIGNALS, signals)
def testWfStepStdioLogAlreadyDownloaded(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
step = WfStep.Create(master_name, builder_name, build_number,
step_name)
step.log_data = self.ABC_TEST_FAILURE_LOG
step.put()
step_log_url = buildbot.CreateStdioLogUrl(master_name, builder_name,
build_number, step_name)
with self.mock_urlfetch() as urlfetch:
urlfetch.register_handler(step_log_url,
'If used, test should fail!')
pipeline = ExtractSignalPipeline(self.FAILURE_INFO)
signals = pipeline.run(self.FAILURE_INFO)
self.assertEqual(self.FAILURE_SIGNALS, signals)
def run(self, master_name, builder_name, build_number):
self._ResetAnalysis(master_name, builder_name, build_number)
# The yield statements below return PipelineFutures, which allow subsequent
# pipelines to refer to previous output values.
# https://github.com/GoogleCloudPlatform/appengine-pipelines/wiki/Python
failure_info = yield DetectFirstFailurePipeline(
master_name, builder_name, build_number)
change_logs = yield PullChangelogPipeline(failure_info)
deps_info = yield ExtractDEPSInfoPipeline(failure_info, change_logs)
signals = yield ExtractSignalPipeline(failure_info)
yield IdentifyCulpritPipeline(failure_info, change_logs, deps_info,
signals)
def testGetTestLevelFailuresInvalid(self):
master_name = 'm'
builder_name = 'b'
build_number = 125
step_name = 'abc_test'
expected_failure_log = 'invalid'
step_log = self._GetGtestResultLog(master_name,
builder_name, build_number, step_name)
failed_test_log = ExtractSignalPipeline._GetReliableTestFailureLog(step_log)
self.assertEqual(expected_failure_log, failed_test_log)
def testGetSignalFromStepLogInvalid(self):
master_name = 'm'
builder_name = 'b'
build_number = 125
step_name = 'abc_test'
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
step_name: {
'last_pass': 124,
'current_failure': 125,
'first_failure': 125,
}
}
}
self.MockGetStdiolog(master_name, builder_name, build_number, step_name)
self.MockGetGtestJsonResult()
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info, False)
step = WfStep.Get(master_name, builder_name, build_number, step_name)
expected_files = {
'content/common/gpu/media/v4l2_video_encode_accelerator.cc': [306]
}
self.assertIsNotNone(step)
self.assertIsNotNone(step.log_data)
self.assertEqual(expected_files, signals['abc_test']['files'])
def testGetTestLevelFailuresInvalid(self):
master_name = 'm'
builder_name = 'b'
build_number = 125
step_name = 'abc_test'
expected_failure_log = 'invalid'
step_log = self._GetGtestResultLog(master_name, builder_name,
build_number, step_name)
failed_test_log = ExtractSignalPipeline._GetReliableTestFailureLog(
step_log)
self.assertEqual(expected_failure_log, failed_test_log)
def testGetSignalFromStepLogInvalid(self):
master_name = 'm'
builder_name = 'b'
build_number = 125
step_name = 'abc_test'
failure_info = {
'master_name': 'm',
'builder_name': 'b',
'build_number': 125,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'abc_test': {
'last_pass': 124,
'current_failure': 125,
'first_failure': 125,
}
}
}
self.MockGetStdiolog(master_name, builder_name, build_number,
step_name)
self.MockGetGtestJsonResult()
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
step = WfStep.Get(master_name, builder_name, build_number, step_name)
expected_files = {
'content/common/gpu/media/v4l2_video_encode_accelerator.cc': [306]
}
self.assertIsNotNone(step)
self.assertIsNotNone(step.log_data)
self.assertEqual(expected_files, signals['abc_test']['files'])
def testGetTestLevelFailures(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
expected_failure_log = ('ERROR:x_test.cc:1234\na/b/u2s1.cc:567: Failure\n'
'ERROR:[2]: 2594735000 bogo-microseconds\n'
'ERROR:x_test.cc:1234\na/b/u2s1.cc:567: Failure\n'
'ERROR:x_test.cc:1234\na/b/u2s1.cc:567: Failure\n'
'a/b/u3s2.cc:110: Failure\n'
'a/b/u3s2.cc:110: Failure\n'
'a/b/u3s2.cc:110: Failure\n'
'a/b/u3s2.cc:110: Failure\n'
)
step_log = self._GetGtestResultLog(
master_name, builder_name, build_number, step_name)
failed_test_log = ExtractSignalPipeline._GetReliableTestFailureLog(step_log)
self.assertEqual(expected_failure_log, failed_test_log)
def testGetTestLevelFailures(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 'abc_test'
expected_failure_log = (
'ERROR:x_test.cc:1234\na/b/u2s1.cc:567: Failure\n'
'[2]: 2594735000 bogo-microseconds\n'
'ERROR:x_test.cc:1234\na/b/u2s1.cc:567: Failure\n'
'ERROR:x_test.cc:1234\na/b/u2s1.cc:567: Failure\n'
'ERROR:x_test.cc:1234\na/b/u2s1.cc:567: Failure\n'
'a/b/u3s2.cc:110: Failure\n'
'a/b/u3s2.cc:110: Failure\n'
'a/b/u3s2.cc:110: Failure\n'
'a/b/u3s2.cc:110: Failure\n')
step_log = self._GetGtestResultLog(master_name, builder_name,
build_number, step_name)
failed_test_log = ExtractSignalPipeline._GetReliableTestFailureLog(
step_log)
self.assertEqual(expected_failure_log, failed_test_log)
def testExtractSignalsForTests(self):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'abc_test': {
'last_pass': 221,
'current_failure': 223,
'first_failure': 222,
'tests': {
'Unittest2.Subtest1': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
},
'Unittest3.Subtest2': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
}
}
}
}
}
step = WfStep.Create(master_name, builder_name, build_number, 'abc_test')
step.isolated = True
step.log_data = (
'{"Unittest2.Subtest1": "RVJST1I6eF90ZXN0LmNjOjEyMzQKYS9iL3UyczEuY2M6N'
'TY3OiBGYWlsdXJlCkVSUk9SOlsyXTogMjU5NDczNTAwMCBib2dvLW1pY3Jvc2Vjb25kcw'
'pFUlJPUjp4X3Rlc3QuY2M6MTIzNAphL2IvdTNzMi5jYzoxMjM6IEZhaWx1cmUK"'
', "Unittest3.Subtest2": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCmEvYi91M3My'
'LmNjOjEyMzogRmFpbHVyZQo="}')
step.put()
expected_signals = {
'abc_test': {
'files': {
'a/b/u2s1.cc': [567],
'a/b/u3s2.cc': [123, 110]
},
'keywords': {},
'tests': {
'Unittest2.Subtest1': {
'files': {
'a/b/u2s1.cc': [567],
'a/b/u3s2.cc': [123]
},
'keywords': {}
},
'Unittest3.Subtest2': {
'files': {
'a/b/u3s2.cc': [110, 123]
},
'keywords': {}
}
}
}
}
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info, False)
self.assertEqual(expected_signals, signals)
def testExtractSignalsForTests(self):
master_name = 'm'
builder_name = 'b'
build_number = 223
failure_info = {
'master_name': master_name,
'builder_name': builder_name,
'build_number': build_number,
'failed': True,
'chromium_revision': 'a_git_hash',
'failed_steps': {
'abc_test': {
'last_pass': 221,
'current_failure': 223,
'first_failure': 222,
'tests': {
'Unittest2.Subtest1': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
},
'Unittest3.Subtest2': {
'current_failure': 223,
'first_failure': 222,
'last_pass': 221
}
}
}
}
}
step = WfStep.Create(master_name, builder_name, build_number, 'abc_test')
step.isolated = True
step.log_data = (
'{"Unittest2.Subtest1": "RVJST1I6eF90ZXN0LmNjOjEyMzQKYS9iL3UyczEuY2M6N'
'TY3OiBGYWlsdXJlCkVSUk9SOlsyXTogMjU5NDczNTAwMCBib2dvLW1pY3Jvc2Vjb25kcw'
'pFUlJPUjp4X3Rlc3QuY2M6MTIzNAphL2IvdTNzMi5jYzoxMjM6IEZhaWx1cmUK"'
', "Unittest3.Subtest2": "YS9iL3UzczIuY2M6MTEwOiBGYWlsdXJlCmEvYi91M3My'
'LmNjOjEyMzogRmFpbHVyZQo="}')
step.put()
expected_signals = {
'abc_test': {
'files': {
'a/b/u2s1.cc': [567],
'a/b/u3s2.cc': [123, 110]
},
'keywords': {},
'tests': {
'Unittest2.Subtest1': {
'files': {
'a/b/u2s1.cc': [567],
'a/b/u3s2.cc': [123]
},
'keywords': {}
},
'Unittest3.Subtest2': {
'files': {
'a/b/u3s2.cc': [110, 123]
},
'keywords': {}
}
}
}
}
self._CreateAndSaveWfAnanlysis(
master_name, builder_name, build_number)
pipeline = ExtractSignalPipeline()
signals = pipeline.run(failure_info)
self.assertEqual(expected_signals, signals)