def testGroupFailuresByRegerssionRange(self): compile_failure_3 = CompileFailure.Create(self.build.key, 'compile', ['c.o'], 'CC') compile_failure_3.put() failures_with_range = [{ 'failure': self.compile_failure_1, 'last_passed_commit': self.commits[5], 'first_failed_commit': self.commits[6], }, { 'failure': self.compile_failure_2, 'last_passed_commit': self.commits[7], 'first_failed_commit': self.commits[8], }, { 'failure': compile_failure_3, 'last_passed_commit': self.commits[5], 'first_failed_commit': self.commits[6], }] expected_result = [ { 'failures': [self.compile_failure_1, compile_failure_3], 'last_passed_commit': self.commits[5], 'first_failed_commit': self.commits[6], }, { 'failures': [self.compile_failure_2], 'last_passed_commit': self.commits[7], 'first_failed_commit': self.commits[8], }, ] result = analysis_util.GroupFailuresByRegerssionRange( failures_with_range) self.assertItemsEqual(expected_result, result)
def testGetMergedFailure(self): failure = CompileFailure.Create(ndb.Key(LuciFailedBuild, 9876543234), 'compile', ['target1.o']) failure.first_failed_build_id = self.build_id failure.put() self.assertEqual(self.target_entities[0], failure.GetMergedFailure())
def setUp(self): super(CompileFailureTest, self).setUp() self.build_id = 9876543210 self.edges = [ (['target1.o'], 'CXX'), (['target2.o'], 'ACTION'), ] build = LuciFailedBuild.Create( luci_project='chromium', luci_bucket='ci', luci_builder='Linux Builder', build_id=self.build_id, legacy_build_number=12345, gitiles_host='chromium.googlesource.com', gitiles_project='chromium/src', gitiles_ref='refs/heads/master', gitiles_id='git_hash', commit_position=65450, status=20, create_time=datetime(2019, 3, 28), start_time=datetime(2019, 3, 28, 0, 1), end_time=datetime(2019, 3, 28, 1), build_failure_type=StepTypeEnum.COMPILE) build.put() self.target_entities = [] for output_targets, rule in self.edges: target = CompileFailure.Create(build.key, 'compile', output_targets, rule) target.put() self.target_entities.append(target)
def testGetMergedFailureWithSavedMergedFailure(self): dummy_merged_failure = CompileFailure.Create( ndb.Key(LuciFailedBuild, 9876543201), 'compile', ['target1.o']) dummy_merged_failure.put() failure = self.target_entities[0] failure.merged_failure_key = dummy_merged_failure.key self.assertEqual(dummy_merged_failure, failure.GetMergedFailure())
def GetFailureEntitiesForABuild(self, build): compile_failure_entities = CompileFailure.query( ancestor=ndb.Key(luci_build.LuciFailedBuild, build.id)).fetch() assert compile_failure_entities, ( 'No compile failure saved in datastore for build {}'.format( build.id)) return compile_failure_entities
def testCompileFailure(self): build_key = ndb.Key('LuciFailedBuild', self.build_id) failures_in_build = CompileFailure.query(ancestor=build_key).fetch() self.assertEqual(2, len(failures_in_build)) self.assertItemsEqual( [{'target1.o'}, {'target2.o'}], [f.GetFailureIdentifier() for f in failures_in_build]) self.assertEqual(self.build_id, failures_in_build[0].build_id)
def testGetFailuresWithMatchingCompileFailureGroupsWithExistingGroup(self): build_id = 8000000000122 build = Build(builder=self.builder, number=122, id=build_id) build.input.gitiles_commit.host = 'gitiles.host.com' build.input.gitiles_commit.project = 'project/name' build.input.gitiles_commit.ref = 'ref/heads/master' build.input.gitiles_commit.id = 'git_sha' last_passed_build_info = { 'id': 8000000000121, 'number': 121, 'commit_id': 'git_sha_121' } first_failures_in_current_build = { 'failures': { 'install packages': { 'output_targets': [ frozenset(['target1']), ], 'last_passed_build': last_passed_build_info, }, }, 'last_passed_build': last_passed_build_info } compile_failure = CompileFailure.Create( self.group_build.key, 'install packages', ['target1'], 'CXX', first_failed_build_id=self.group_build_id, last_passed_build_id=8000000000160) compile_failure.put() CompileFailureGroup.Create( luci_project=self.context.luci_project_name, luci_bucket=build.builder.bucket, build_id=self.group_build_id, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, last_passed_gitiles_id=last_passed_build_info['commit_id'], last_passed_commit_position=654321, first_failed_gitiles_id=self.first_failed_commit_id, first_failed_commit_position=654340, compile_failure_keys=[compile_failure.key]).put() expected_failures_with_existing_group = { 'install packages': { frozenset(['target1']): self.group_build_id } } self.assertEqual( expected_failures_with_existing_group, ChromeOSProjectAPI().GetFailuresWithMatchingCompileFailureGroups( self.context, build, first_failures_in_current_build))
def _GetCompileFailureEntitiesForABuild(build): build_entity = luci_build.LuciFailedBuild.get_by_id(build.id) assert build_entity, 'No LuciFailedBuild entity for build {}'.format(build.id) compile_failure_entities = CompileFailure.query( ancestor=build_entity.key).fetch() assert compile_failure_entities, ( 'No compile failure saved in datastore for build {}'.format(build.id)) return compile_failure_entities
def testGetCulpritsForFailures(self): culprit = CulpritNdb.Create(self.gitiles_host, self.gitiles_project, self.gitiles_ref, 'git_hash_123', 123) culprit.put() failure1 = CompileFailure.Create(self.build.key, 'compile', ['a.o'], 'CC') failure1.culprit_commit_key = culprit.key failure1.put() failure2 = CompileFailure.Create(self.build.key, 'compile', ['b.o'], 'CC') failure2.culprit_commit_key = culprit.key failure2.put() culprits = analysis_util.GetCulpritsForFailures([failure1, failure2]) self.assertEqual(1, len(culprits)) self.assertEqual('git_hash_123', culprits[0].commit.id)
def testOnCompileFailureAnalysisResultRequestedAnalysisRunning(self): build_id = 800000000123 request = findit_result.BuildFailureAnalysisRequest( build_id=build_id, failed_steps=[self.compile_step_name]) build = LuciFailedBuild.Create( luci_project='chromium', luci_bucket='ci', luci_builder='Linux Builder', build_id=build_id, legacy_build_number=12345, gitiles_host='chromium.googlesource.com', gitiles_project='chromium/src', gitiles_ref='refs/heads/master', gitiles_id='git_hash', commit_position=65450, status=20, create_time=datetime(2019, 3, 28), start_time=datetime(2019, 3, 28, 0, 1), end_time=datetime(2019, 3, 28, 1), build_failure_type=StepTypeEnum.COMPILE) build.put() compile_failure = CompileFailure.Create(build.key, self.compile_step_name, ['target1'], 'CXX') compile_failure.first_failed_build_id = build.build_id compile_failure.failure_group_build_id = build.build_id compile_failure.put() analysis = CompileFailureAnalysis.Create( luci_project=self.context.luci_project_name, luci_bucket='postsubmit', luci_builder='Linux Builder', build_id=build_id, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, last_passed_gitiles_id='last_passed_git_hash', last_passed_commit_position=65430, first_failed_gitiles_id='git_hash', first_failed_commit_position=65450, rerun_builder_id='chromeos/postsubmit/builder-bisect', compile_failure_keys=[]) analysis.status = analysis_status.RUNNING analysis.Save() responses = compile_api.OnCompileFailureAnalysisResultRequested( request, build) self.assertEqual(1, len(responses)) self.assertEqual(0, len(responses[0].culprits)) self.assertFalse(responses[0].is_finished) self.assertTrue(responses[0].is_supported)
def testGetMergedFailureForNonFirstFailure(self): first_failed_build_id = 9876543201 dummy_merged_failure = CompileFailure.Create( ndb.Key(LuciFailedBuild, first_failed_build_id), 'compile', ['target1.o'], first_failed_build_id=first_failed_build_id, failure_group_build_id=first_failed_build_id) dummy_merged_failure.put() failure = self.target_entities[0] failure.first_failed_build_id = first_failed_build_id self.assertEqual(dummy_merged_failure, failure.GetMergedFailure())
def testSaveCompileFailuresOnlyStepLevelFailures(self, _): detailed_compile_failures = { 'compile': { 'failures': {}, 'first_failed_build': { 'id': 8000000000121, 'number': 121, 'commit_id': 'git_sha' }, 'last_passed_build': { 'id': 8000000000120, 'number': 120, 'commit_id': 'git_sha' }, }, } # Prepares data for first failed build. first_failed_build = self._MockBuild(8000000000121, 121, 'git_sha_121') first_failed_build_entity = luci_build.SaveFailedBuild( self.context, first_failed_build, StepTypeEnum.COMPILE) first_failure = CompileFailure.Create(first_failed_build_entity.key, 'compile', None, 'CXX') first_failure.put() pre_compile_analysis.SaveCompileFailures(self.context, self.build, detailed_compile_failures) build_entity = LuciFailedBuild.get_by_id(self.build_id) self.assertIsNotNone(build_entity) compile_failures = CompileFailure.query( ancestor=build_entity.key).fetch() self.assertEqual(1, len(compile_failures)) self.assertEqual(8000000000121, compile_failures[0].first_failed_build_id) self.assertEqual([], compile_failures[0].output_targets) self.assertEqual(first_failure.key, compile_failures[0].merged_failure_key)
def OnCompileFailureAnalysisResultRequested(request, requested_build): """Returns the findings for the requested build's compile failure. Since SoM doesn't have atomic failure info for compile steps, currently Findit will only respond with aggregated step level results. Args: request(findit_result.BuildFailureAnalysisRequest): request for a build failure. requested_build(LuciFailedBuild): A LuciFailedBuild entity with COMPILE build_failure_type. Returns: [findit_result.BuildFailureAnalysisResponse]: Analysis results for the requested build. """ compile_failures = CompileFailure.query( ancestor=requested_build.key).fetch() if not compile_failures: return None requested_steps = request.failed_steps requested_failures = defaultdict(list) for failure in compile_failures: if requested_steps and failure.step_ui_name not in requested_steps: continue requested_failures[failure.step_ui_name].append(failure) responses = [] for step_ui_name, requested_failures_in_step in requested_failures.iteritems( ): merged_failures = [] for failure in requested_failures_in_step: # Merged failures are the failures being actually analyzed and only they # have stored culprits info. merged_failures.append(failure.GetMergedFailure()) culprits = CompileAnalysisAPI().GetCulpritsForFailures(merged_failures) response = BuildFailureAnalysisResponse( build_id=request.build_id, build_alternative_id=request.build_alternative_id, step_name=step_ui_name, test_name=None, culprits=culprits, is_finished=_IsAnalysisFinished(merged_failures), is_supported=True, ) responses.append(response) return responses
def setUp(self): super(AnalysisUtilTest, self).setUp() self.gitiles_host = 'gitiles.host.com' self.gitiles_project = 'project/name' self.gitiles_ref = 'ref/heads/master' self.build = LuciFailedBuild.Create( luci_project='chromium', luci_bucket='ci', luci_builder='Linux Builder', build_id=9876543210, legacy_build_number=12345, gitiles_host='chromium.googlesource.com', gitiles_project='chromium/src', gitiles_ref='refs/heads/master', gitiles_id='git_hash', commit_position=65450, status=20, create_time=datetime(2019, 3, 28), start_time=datetime(2019, 3, 28, 0, 1), end_time=datetime(2019, 3, 28, 1), build_failure_type=StepTypeEnum.COMPILE) self.build.put() self.compile_failure_1 = CompileFailure.Create(self.build.key, 'compile', ['a.o'], 'CC') self.compile_failure_1.put() self.compile_failure_2 = CompileFailure.Create(self.build.key, 'compile', ['b.o'], 'CC') self.compile_failure_2.put() self.commits = [] for i in xrange(0, 11): self.commits.append(self._CreateGitilesCommit('r%d' % i, 100 + i))
def _CreateFailure(self, failed_build_key, step_ui_name, first_failed_build_id, last_passed_build_id, merged_failure_key, atomic_failure, properties): """Creates a CompileFailure entity.""" return CompileFailure.Create( failed_build_key=failed_build_key, step_ui_name=step_ui_name, output_targets=list(atomic_failure or []), rule=(properties or {}).get('rule'), first_failed_build_id=first_failed_build_id, last_passed_build_id=last_passed_build_id, # Default to first_failed_build_id, will be updated later if matching # group exists. failure_group_build_id=first_failed_build_id, merged_failure_key=merged_failure_key)
def testGetFirstFailuresInCurrentBuildWithoutGroup(self, *_): build_121_info = { 'id': 8000000000121, 'number': self.build_number - 2, 'commit_id': 'git_sha_121' } first_failures_in_current_build = { 'failures': { 'compile': { 'output_targets': [frozenset(['target1']), frozenset(['target2'])], 'last_passed_build': build_121_info, }, }, 'last_passed_build': build_121_info } # Creates and saves entities of the existing group. detailed_compile_failures = { 'compile': { 'failures': { frozenset(['target1']): { 'rule': 'CXX', 'first_failed_build': self.build_info, 'last_passed_build': build_121_info, }, frozenset(['target2']): { 'rule': 'ACTION', 'first_failed_build': self.build_info, 'last_passed_build': build_121_info, }, }, 'first_failed_build': self.build_info, 'last_passed_build': build_121_info, }, } pre_compile_analysis.SaveCompileFailures(self.context, self.build, detailed_compile_failures) # Prepares data for existing failure group. group_build = self._MockBuild(8000000000134, 12134, 'git_sha_134', builder_name='Mac') group_build_entity = luci_build.SaveFailedBuild( self.context, group_build, StepTypeEnum.COMPILE) group_failure1 = CompileFailure.Create(group_build_entity.key, 'compile', ['target1'], 'CXX') group_failure1.put() group_failure2 = CompileFailure.Create(group_build_entity.key, 'compile', ['target2'], 'ACTION') group_failure2.put() self.assertEqual( { 'failures': {}, 'last_passed_build': None }, pre_compile_analysis.GetFirstFailuresInCurrentBuildWithoutGroup( self.context, self.build, first_failures_in_current_build)) build = LuciFailedBuild.get_by_id(self.build_id) compile_failures = CompileFailure.query(ancestor=build.key).fetch() self.assertEqual(2, len(compile_failures)) for failure in compile_failures: if failure.output_targets == ['target1']: self.assertEqual(group_failure1.key, failure.merged_failure_key) else: self.assertEqual(group_failure2.key, failure.merged_failure_key)
def SaveCompileFailures(context, build, detailed_compile_failures): """Saves the failed build and compile failures in data store. Args: context (findit_v2.services.context.Context): Scope of the analysis. build (buildbucket build.proto): ALL info about the build. detailed_compile_failures (dict): A dict of detailed compile failures. { 'step_name': { 'failures': { frozenset(['target1', 'target2']): { 'rule': 'emerge', 'first_failed_build': { 'id': 8765432109, 'number': 123, 'commit_id': 654321 }, 'last_passed_build': None }, ... }, 'first_failed_build': { 'id': 8765432109, 'number': 123, 'commit_id': 654321 }, 'last_passed_build': None }, } """ build_entity = luci_build.SaveFailedBuild(context, build, StepTypeEnum.COMPILE) assert build_entity, 'Failed to create failure entity for build {}'.format( build.id) failed_build_key = build_entity.key compile_failure_entities = [] first_failures = {} for step_ui_name, step_info in detailed_compile_failures.iteritems(): failures = step_info['failures'] if not failures: logging.warning( 'Cannot get detailed compile failure info for build %d,' ' saving step level info only.', build.id) first_failed_build_id = step_info.get('first_failed_build', {}).get('id') merged_failure_key = compile_failure.GetMergedFailureKey( first_failures, first_failed_build_id, step_ui_name, None) new_entity = CompileFailure.Create( failed_build_key=failed_build_key, step_ui_name=step_ui_name, output_targets=None, first_failed_build_id=first_failed_build_id, last_passed_build_id=step_info.get('last_passed_build', {}).get('id'), # Default to first_failed_build_id, will be updated later if matching # group exists. failure_group_build_id=first_failed_build_id, merged_failure_key=merged_failure_key) compile_failure_entities.append(new_entity) continue for output_targets, failure in failures.iteritems(): first_failed_build_id = failure.get('first_failed_build', {}).get('id') merged_failure_key = compile_failure.GetMergedFailureKey( first_failures, first_failed_build_id, step_ui_name, output_targets) new_entity = CompileFailure.Create( failed_build_key=failed_build_key, step_ui_name=step_ui_name, output_targets=list(output_targets), first_failed_build_id=first_failed_build_id, last_passed_build_id=(failure.get('last_passed_build') or {}).get('id'), # Default to first_failed_build_id, will be updated later if matching # group exists. failure_group_build_id=first_failed_build_id, rule=failure.get('rule'), dependencies=failure.get('dependencies'), merged_failure_key=merged_failure_key) compile_failure_entities.append(new_entity) ndb.put_multi(compile_failure_entities)
def _GetMergedFailureKey(self, failure_entities, referred_build_id, step_ui_name, atomic_failure): return CompileFailure.GetMergedFailureKey(failure_entities, referred_build_id, step_ui_name, atomic_failure)
def testGetFailuresWithMatchingCompileFailureGroupsFailureNotExactlySame( self): build_id = 8000000000122 build = Build(builder=self.builder, number=122, id=build_id) build.input.gitiles_commit.host = 'gitiles.host.com' build.input.gitiles_commit.project = 'project/name' build.input.gitiles_commit.ref = 'ref/heads/master' build.input.gitiles_commit.id = 'git_sha' output_target1 = json.dumps({ 'category': 'chromeos-base', 'packageName': 'target1' }) output_target2 = json.dumps({ 'category': 'chromeos-base', 'packageName': 'target2' }) last_passed_build_info = { 'id': 8000000000121, 'number': 121, 'commit_id': 'git_sha_121' } first_failures_in_current_build = { 'failures': { 'install packages': { 'output_targets': [frozenset([output_target1]), frozenset([output_target2])], 'last_passed_build': last_passed_build_info, }, }, 'last_passed_build': last_passed_build_info } compile_failure = CompileFailure.Create( self.group_build.key, 'install packages', [output_target1], 'CXX', first_failed_build_id=self.group_build_id, last_passed_build_id=8000000000160) compile_failure.put() CompileFailureGroup.Create( luci_project=self.context.luci_project_name, luci_bucket=build.builder.bucket, build_id=self.group_build_id, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, last_passed_gitiles_id=last_passed_build_info['commit_id'], last_passed_commit_position=654321, first_failed_gitiles_id=self.first_failed_commit_id, first_failed_commit_position=654340, compile_failure_keys=[compile_failure.key]).put() self.assertEqual( {}, ChromeOSProjectAPI().GetFailuresWithMatchingCompileFailureGroups( self.context, build, first_failures_in_current_build))
def testSaveCompileFailures(self, _): detailed_compile_failures = { 'compile': { 'failures': { frozenset(['target1', 'target2']): { 'rule': 'CXX', 'first_failed_build': { 'id': 8000000000121, 'number': 121, 'commit_id': 'git_sha_121' }, 'last_passed_build': { 'id': 8000000000120, 'number': 120, 'commit_id': 'git_sha' }, }, }, 'first_failed_build': { 'id': 8000000000121, 'number': 121, 'commit_id': 'git_sha_121' }, 'last_passed_build': { 'id': 8000000000120, 'number': 120, 'commit_id': 'git_sha' }, }, } # Prepares data for existing failure group. group_build = self._MockBuild(8000003400121, 12134, 'git_sha_121', builder_name='Mac') group_build_entity = luci_build.SaveFailedBuild( self.context, group_build, StepTypeEnum.COMPILE) group_failure = CompileFailure.Create(group_build_entity.key, 'compile', ['target1', 'target2'], 'CXX') group_failure.put() # Prepares data for first failed build. first_failed_build = self._MockBuild(8000000000121, 121, 'git_sha_121') first_failed_build_entity = luci_build.SaveFailedBuild( self.context, first_failed_build, StepTypeEnum.COMPILE) first_failure = CompileFailure.Create(first_failed_build_entity.key, 'compile', ['target1', 'target2'], 'CXX') first_failure.merged_failure_key = group_failure.key first_failure.put() pre_compile_analysis.SaveCompileFailures(self.context, self.build, detailed_compile_failures) build = LuciFailedBuild.get_by_id(self.build_id) self.assertIsNotNone(build) compile_failures = CompileFailure.query(ancestor=build.key).fetch() self.assertEqual(1, len(compile_failures)) self.assertEqual(8000000000121, compile_failures[0].first_failed_build_id) self.assertEqual(group_failure.key, compile_failures[0].merged_failure_key)
def setUp(self): super(CompileFailureRerunAnalysisTest, self).setUp() self.build_id = 8000000000123 self.build_number = 123 self.builder = BuilderID(project='chromium', bucket='ci', builder='linux-rel') self.build = Build(id=self.build_id, builder=self.builder, number=self.build_number, status=common_pb2.FAILURE) self.build.input.gitiles_commit.host = 'gitiles.host.com' self.build.input.gitiles_commit.project = 'project/name' self.build.input.gitiles_commit.ref = 'ref/heads/master' self.build.input.gitiles_commit.id = 'git_sha_123' self.build.create_time.FromDatetime(datetime(2019, 4, 9)) self.build.start_time.FromDatetime(datetime(2019, 4, 9, 0, 1)) self.build.end_time.FromDatetime(datetime(2019, 4, 9, 1)) self.context = Context(luci_project_name='chromium', gitiles_host='gitiles.host.com', gitiles_project='project/name', gitiles_ref='ref/heads/master', gitiles_id='git_sha') build_entity = LuciFailedBuild.Create( luci_project=self.context.luci_project_name, luci_bucket=self.build.builder.bucket, luci_builder=self.build.builder.builder, build_id=self.build_id, legacy_build_number=self.build_number, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id=self.context.gitiles_id, commit_position=6000005, status=20, create_time=datetime(2019, 3, 28), start_time=datetime(2019, 3, 28, 0, 1), end_time=datetime(2019, 3, 28, 1), build_failure_type=StepTypeEnum.COMPILE) build_entity.put() self.compile_failure = CompileFailure.Create( failed_build_key=build_entity.key, step_ui_name='compile', output_targets=['a.o'], first_failed_build_id=self.build_id, failure_group_build_id=None) self.compile_failure.put() self.analysis = CompileFailureAnalysis.Create( luci_project=self.context.luci_project_name, luci_bucket=self.build.builder.bucket, luci_builder=self.build.builder.builder, build_id=self.build_id, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, last_passed_gitiles_id='left_sha', last_passed_commit_position=6000000, first_failed_gitiles_id=self.context.gitiles_id, first_failed_commit_position=6000005, rerun_builder_id='chromium/findit/findit-variables', compile_failure_keys=[self.compile_failure.key]) self.analysis.Save()
def testGetMergedFailureKey(self): self.assertEqual( self.target_entities[0].key, CompileFailure.GetMergedFailureKey( {}, self.build_id, 'compile', self.target_entities[0].GetFailureIdentifier()))
def testGetMergedFailureKeyNoBuildId(self): with self.assertRaises(AssertionError): CompileFailure.GetMergedFailureKey({}, None, 's', frozenset([]))