def testSearchV2BuildsOnBuilderWithBuildRange(self, mock_post): builder = BuilderID(project='chromium', bucket='try', builder='linux-rel') mock_build = Build(number=100) mock_response = SearchBuildsResponse( next_page_token='next_page_token', builds=[mock_build]) mock_headers = {'X-Prpc-Grpc-Code': '0'} binary_data = mock_response.SerializeToString() mock_post.return_value = (200, binary_data, mock_headers) res = buildbucket_client.SearchV2BuildsOnBuilder( builder, build_range=(None, 800000000099), page_size=1) self.assertEqual('next_page_token', res.next_page_token) self.assertEqual(1, len(res.builds)) self.assertEqual(100, res.builds[0].number)
def testProcessRerunBuildResultNoEntity(self): build_id = 8000000000123 build_number = 123 builder = BuilderID( project='chromeos', bucket='postsubmit', builder='findit-variable') build = Build( id=build_id, builder=builder, number=build_number, status=common_pb2.FAILURE, tags=[{ 'key': 'analyzed_build_id', 'value': str(self.analyzed_build_id) }]) self.assertFalse( test_analysis.OnTestRerunBuildCompletion(self.context, build))
def testProcessAndSaveRerunBuildResultAnalysisMissing(self): build_id = 8000000000123 build_number = 123 builder = BuilderID(project='chromeos', bucket='postsubmit', builder='findit-variable') build = Build(id=build_id, builder=builder, number=build_number, status=common_pb2.FAILURE, tags=[{ 'key': 'analyzed_build_id', 'value': '87654321' }]) self.assertFalse( compile_api.OnCompileRerunBuildCompletion(self.context, build))
def testProcessRerunBuildResultBuildPassed(self, mock_compile_failures, mock_analysis): build_id = 8000000000123 build_number = 123 builder = BuilderID(project='chromeos', bucket='postsubmit', builder='findit-variable') build = Build(id=build_id, builder=builder, number=build_number, status=common_pb2.SUCCESS, tags=[{ 'key': 'analyzed_build_id', 'value': str(self.analyzed_build_id) }]) build.input.gitiles_commit.host = 'gitiles.host.com' build.input.gitiles_commit.project = 'project/name' build.input.gitiles_commit.ref = 'ref/heads/master' build.input.gitiles_commit.id = 'git_sha_6543221' build.create_time.FromDatetime(datetime(2019, 4, 9)) step1 = Step(name='s1', status=common_pb2.SUCCESS) step2 = Step(name=self.compile_step_name, status=common_pb2.SUCCESS) build.steps.extend([step1, step2]) CompileRerunBuild.Create(luci_project=self.context.luci_project_name, luci_bucket=build.builder.bucket, luci_builder=build.builder.builder, build_id=build_id, legacy_build_number=build_number, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id='git_sha_6543221', commit_position=6543221, status=build.status, create_time=build.create_time.ToDatetime(), parent_key=self.analysis.key).put() self.assertTrue( compile_analysis.OnCompileRerunBuildCompletion( self.context, build)) self.assertFalse(mock_compile_failures.called) rerun_build = CompileRerunBuild.get_by_id(build_id, parent=self.analysis.key) self.assertEqual({}, rerun_build.GetFailuresInBuild()) self.assertTrue(mock_analysis.called)
def testGetLatestCommitPositionWithBuild(self, mocked_build, _): master_name = 'm' builder_name = 'b' target_name = 't' expected_commit_position = 100000 expected_revision = 'r100000' build = Build(builder=BuilderID( project='chromium', bucket='ci', builder=builder_name)) build.input.gitiles_commit.project = 'chromium/src' build.input.gitiles_commit.host = 'chromium.googlesource.com' build.input.gitiles_commit.ref = 'refs/heads/master' build.input.gitiles_commit.id = expected_revision mocked_build.return_value = SearchBuildsResponse(builds=[build]) self.assertEqual((expected_commit_position, expected_revision), build_util.GetLatestCommitPositionAndRevision( master_name, builder_name, target_name))
def testSaveFailedBuild(self, _): builder = BuilderID(project='chromium', bucket='try', builder='linux-rel') build = Build( id=87654321, builder=builder, number=123, status=common_pb2.FAILURE) build.create_time.FromDatetime(datetime(2019, 4, 9)) build.start_time.FromDatetime(datetime(2019, 4, 9, 0, 1)) build.end_time.FromDatetime(datetime(2019, 4, 9, 1)) context = Context( luci_project_name='project', gitiles_host='gitiles.host.com', gitiles_project='project/name', gitiles_ref='ref/heads/master', gitiles_id='git_sha') build_entity = luci_build.SaveFailedBuild(context, build, StepTypeEnum.COMPILE) self.assertIsNotNone(build_entity)
def ParseBuilderId(builder_id): """Returns the builder config in a dict from a builder_id. Args: builder_id (str): Builder id in the format project/bucket/builder. Returns: build_pb2.BuilderID based on the str formatted builder_id. """ parts = builder_id.split('/') assert len(parts) == 3, 'builder_id {} in an invalid format.'.format( builder_id) return BuilderID( project=parts[0], bucket=parts[1], builder=parts[2], )
def setUp(self): super(ChromeOSProjectAPITest, self).setUp() self.first_failed_commit_id = 'git_sha' self.first_failed_commit_position = 65450 self.context = Context(luci_project_name='chromeos', gitiles_host='gitiles.host.com', gitiles_project='project/name', gitiles_ref='ref/heads/master', gitiles_id='git_sha') self.builder = BuilderID(project='chromeos', bucket='postsubmit', builder='builder-postsubmit') self.group_build_id = 8000000000189 self.group_build = LuciFailedBuild.Create( luci_project=self.context.luci_project_name, luci_bucket=self.builder.bucket, luci_builder='builder2-postsubmit', build_id=self.group_build_id, legacy_build_number=12345, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id=self.context.gitiles_id, commit_position=self.first_failed_commit_position, status=20, create_time=datetime(2019, 3, 28), start_time=datetime(2019, 3, 28, 0, 1), end_time=datetime(2019, 3, 28, 1), build_failure_type=StepTypeEnum.COMPILE) self.group_build.put() self.output_target1 = json.dumps({ 'category': 'chromeos-base', 'packageName': 'target1' }) self.output_target2 = json.dumps({ 'category': 'chromeos-base', 'packageName': 'target2' }) self.output_targets = [self.output_target1, self.output_target2]
def testTriggerRerunBuildFailedToGetProperty(self, mock_trigger_build, _): """This test is for the case where there's already an existing rerun build, so no new rerun-build should be scheduled.""" rerun_commit = GitilesCommit( gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id='6000002', commit_position=6000002) rerun_builder = BuilderID(project='chromium', bucket='findit', builder='findit-variables') output_targets = {'compile': ['a.o']} compile_failure_rerun_analysis.TriggerRerunBuild( self.context, self.build_id, self.build, self.analysis.key, rerun_builder, rerun_commit, output_targets) self.assertFalse(mock_trigger_build.called)
def setUp(self): super(PreCompileAnalysisTest, self).setUp() self.build_id = 8000000000123 self.build_number = 123 self.builder = BuilderID(project='chromium', bucket='ci', builder='Linux Tests') self.build = self._MockBuild(self.build_id, self.build_number, 'git_sha_123') self.context = Context(luci_project_name='chromium', gitiles_host='gitiles.host.com', gitiles_project='project/name', gitiles_ref='ref/heads/master', gitiles_id='git_sha_123') self.build_info = { 'id': 8000000000123, 'number': self.build_number, 'commit_id': 'git_sha_123' }
def _MockBuild(self, build_id, build_number, gitiles_commit_id, builder_name='Linux Tests', build_status=common_pb2.FAILURE): builder = BuilderID(project='chromium', bucket='ci', builder=builder_name) build = Build(id=build_id, builder=builder, number=build_number, status=build_status) build.input.gitiles_commit.host = 'gitiles.host.com' build.input.gitiles_commit.project = 'project/name' build.input.gitiles_commit.ref = 'ref/heads/master' build.input.gitiles_commit.id = gitiles_commit_id build.create_time.FromDatetime(datetime(2019, 4, 9)) build.start_time.FromDatetime(datetime(2019, 4, 9, 0, 1)) build.end_time.FromDatetime(datetime(2019, 4, 9, 1)) return build
def testTriggerRerunBuild(self, mock_trigger_build, _): new_build_id = 800000024324 new_build = Build(id=new_build_id, number=300) new_build.status = common_pb2.SCHEDULED new_build.create_time.FromDatetime(datetime(2019, 4, 20)) rerun_builder = BuilderID(project='chromium', bucket='findit', builder='findit-variables') rerun_commit = GitilesCommit( gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id='6000002', commit_position=6000002) output_targets = {'compile': ['a.o']} mock_trigger_build.return_value = new_build compile_failure_rerun_analysis.TriggerRerunBuild( self.context, self.build_id, self.build, self.analysis.key, rerun_builder, rerun_commit, output_targets) rerun_build = CompileRerunBuild.get_by_id(new_build_id, parent=self.analysis.key) self.assertIsNotNone(rerun_build) mock_trigger_build.assert_called_once_with( rerun_builder, common_pb2.GitilesCommit(project=rerun_commit.gitiles_project, host=rerun_commit.gitiles_host, ref=rerun_commit.gitiles_ref, id=rerun_commit.gitiles_id), {'recipe': 'compile'}, tags=[{ 'value': 'compile-failure-culprit-finding', 'key': 'purpose' }, { 'value': str(self.build.id), 'key': 'analyzed_build_id' }])
def GetV2BuildByBuilderAndBuildNumber(project, bucket, builder_name, build_number, fields=None): """Get a buildbucket build from the v2 API by build info.""" builder = BuilderID(project=project, bucket=bucket, builder=builder_name) request = GetBuildRequest(builder=builder, build_number=build_number, fields=fields) status_code, content, response_headers = FinditHttpClient().Post( _BUILDBUCKET_V2_GET_BUILD_ENDPOINT, request.SerializeToString(), headers={'Content-Type': 'application/prpc; encoding=binary'}) if status_code == 200 and response_headers.get( 'X-Prpc-Grpc-Code') == GRPC_OK: result = Build() result.ParseFromString(content) return result logging.warning('Unexpected prpc code: %s', response_headers.get('X-Prpc-Grpc-Code')) return None
def IteratePreviousBuildsFrom(master_name, builder_name, build_id, entry_limit): luci_project, luci_bucket = buildbot.GetLuciProjectAndBucketForMaster( master_name) builder = BuilderID(project=luci_project, bucket=luci_bucket, builder=builder_name) entry_number = 0 # End_build_id in build_range when query the previous build. end_build_id = build_id while entry_number <= entry_limit: # pragma: no branch. search_builds_response = buildbucket_client.SearchV2BuildsOnBuilder( builder, build_range=(None, end_build_id), page_size=2, fields=FieldMask(paths=['builds.*.*'])) if not search_builds_response.builds: # No more previous build. return previous_build = None # TODO(crbug.com/969124): remove the loop when SearchBuilds RPC works as # expected. for search_build in search_builds_response.builds: if search_build.id != end_build_id: previous_build = search_build break if not previous_build: logging.warning('No previous build found for build %d.', end_build_id) return end_build_id = previous_build.id entry_number += 1 yield previous_build
def testTriggerV2Build(self, mock_post): mock_build = Build() mock_headers = {'X-Prpc-Grpc-Code': '0'} binary_data = mock_build.SerializeToString() mock_post.return_value = (200, binary_data, mock_headers) builder = BuilderID(project='chromium', bucket='try', builder='linux-rel') gitiles_commit = GitilesCommit( project='gitiles/project', host='gitiles.host.com', ref='refs/heads/master', id='git_hash') properties = { 'property1': 'property1', 'property2': ['property2'], 'property3': { 'property3-key': 'property3-value' } } build = buildbucket_client.TriggerV2Build(builder, gitiles_commit, properties) self.assertIsNotNone(build)
def testGetFailedStepsInBuild(self): build_id = 8000000000123 build_number = 123 builder = BuilderID(project='chromium', bucket='try', builder='linux-rel') build = Build(id=build_id, builder=builder, number=build_number, status=common_pb2.FAILURE) step1 = Step(name='s1', status=common_pb2.SUCCESS) step2 = Step(name='compile', status=common_pb2.FAILURE) build.steps.extend([step1, step2]) context = Context(luci_project_name='chromium', gitiles_host='gitiles.host.com', gitiles_project='project/name', gitiles_ref='ref/heads/master', gitiles_id='git_sha') failed_steps = build_util.GetFailedStepsInBuild(context, build) self.assertEqual(1, len(failed_steps)) self.assertEqual('compile', failed_steps[0][0].name) self.assertEqual(StepTypeEnum.COMPILE, failed_steps[0][1])
def testProcessRerunBuildResult(self, mock_test_failures): build_id = 8000000000123 build_number = 123 builder = BuilderID( project='chromeos', bucket='postsubmit', builder='findit-variable') build = Build( id=build_id, builder=builder, number=build_number, status=common_pb2.FAILURE, tags=[{ 'key': 'analyzed_build_id', 'value': str(self.analyzed_build_id) }]) build.input.gitiles_commit.host = 'gitiles.host.com' build.input.gitiles_commit.project = 'project/name' build.input.gitiles_commit.ref = 'ref/heads/master' build.input.gitiles_commit.id = 'git_sha_6543221' build.create_time.FromDatetime(datetime(2019, 4, 9)) build.end_time.FromDatetime(datetime(2019, 4, 9, 0, 30)) step1 = Step(name='s1', status=common_pb2.SUCCESS) step2 = Step(name=self.test_step_name, status=common_pb2.FAILURE) build.steps.extend([step1, step2]) build.output.properties['test_failures'] = { 'xx_test_failures': [{ 'failed_step': self.test_step_name, 'test_spec': 'test_spec' },], } mock_test_failures.return_value = { self.test_step_name: { 'failures': {}, 'first_failed_build': { 'id': build_id, 'number': build_number, 'commit_id': 'git_sha_6543221' }, 'last_passed_build': None, 'properties': {} }, } TestRerunBuild.Create( luci_project=self.context.luci_project_name, luci_bucket=build.builder.bucket, luci_builder=build.builder.builder, build_id=build_id, legacy_build_number=build_number, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id='git_sha_6543221', commit_position=6543221, status=build.status, create_time=build.create_time.ToDatetime(), parent_key=self.analysis.key).put() self.assertTrue( test_analysis._ProcessAndSaveRerunBuildResult( self.context, self.analyzed_build_id, build)) rerun_build = TestRerunBuild.get_by_id(build_id, parent=self.analysis.key) self.assertItemsEqual({ self.test_step_name: [] }, rerun_build.GetFailuresInBuild()) self.assertEqual(datetime(2019, 4, 9, 0, 30), rerun_build.end_time)
def testProcessRerunBuildResult(self, mock_compile_failures): build_id = 8000000000123 build_number = 123 builder = BuilderID(project='chromeos', bucket='postsubmit', builder='findit-variable') build = Build(id=build_id, builder=builder, number=build_number, status=common_pb2.FAILURE, tags=[{ 'key': 'analyzed_build_id', 'value': str(self.analyzed_build_id) }]) build.input.gitiles_commit.host = 'gitiles.host.com' build.input.gitiles_commit.project = 'project/name' build.input.gitiles_commit.ref = 'ref/heads/master' build.input.gitiles_commit.id = 'git_sha_6543221' build.create_time.FromDatetime(datetime(2019, 4, 9)) step1 = Step(name='s1', status=common_pb2.SUCCESS) step2 = Step(name=self.compile_step_name, status=common_pb2.FAILURE) build.steps.extend([step1, step2]) output_target = json.dumps({ 'category': 'chromeos-base', 'packageName': 'target2' }) build.output.properties['build_compile_failure_output'] = { 'failures': [ { 'output_targets': [output_target], 'rule': 'emerge' }, ], 'failed_step': self.compile_step_name } mock_compile_failures.return_value = { self.compile_step_name: { 'failures': { frozenset(['target1.o', 'target2.o']): { 'rule': 'CXX', 'first_failed_build': { 'id': build_id, 'number': build_number, 'commit_id': 'git_sha_6543221' }, 'last_passed_build': None }, }, 'first_failed_build': { 'id': build_id, 'number': build_number, 'commit_id': 'git_sha_6543221' }, 'last_passed_build': None }, } CompileRerunBuild.Create(luci_project=self.context.luci_project_name, luci_bucket=build.builder.bucket, luci_builder=build.builder.builder, build_id=build_id, legacy_build_number=build_number, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id='git_sha_6543221', commit_position=6543221, status=build.status, create_time=build.create_time.ToDatetime(), parent_key=self.analysis.key).put() self.assertTrue( compile_api._ProcessAndSaveRerunBuildResult( self.context, self.analyzed_build_id, build)) rerun_build = CompileRerunBuild.get_by_id(build_id, parent=self.analysis.key) self.assertItemsEqual( ['target1.o', 'target2.o'], rerun_build.GetFailedTargets()[self.compile_step_name])
def testProcessBuildForFlakes(self, mock_metadata, mock_build, mock_normalized_test_name, mock_lable_name, *_): flake_type_enum = FlakeType.CQ_FALSE_REJECTION build_id = 123 luci_project = 'luci_project' luci_bucket = 'luci_bucket' luci_builder = 'luci_builder' legacy_master_name = 'legacy_master_name' start_time = datetime(2019, 3, 6) end_time = datetime(2019, 3, 6, 0, 0, 10) findit_step = Step() findit_step.name = 'FindIt Flakiness' step1 = Step() step1.name = 'step1 (with patch)' step1.start_time.FromDatetime(start_time) step1.end_time.FromDatetime(end_time) builder = BuilderID( project=luci_project, bucket=luci_bucket, builder=luci_builder, ) build = Build(id=build_id, builder=builder, number=build_id) build.steps.extend([findit_step, step1]) build.input.properties['mastername'] = legacy_master_name build.input.properties['patch_project'] = 'chromium/src' mock_change = build.input.gerrit_changes.add() mock_change.host = 'mock.gerrit.host' mock_change.change = 12345 mock_change.patchset = 1 mock_build.return_value = build def _MockTestName(test_name, _step_ui_name): # pylint: disable=unused-argument return test_name mock_normalized_test_name.side_effect = _MockTestName mock_lable_name.side_effect = _MockTestName flakiness_metadata = { 'Failing With Patch Tests That Caused Build Failure': { 'step1 (with patch)': ['s1_t1', 's1_t2'] }, 'Step Layer Flakiness': {} } mock_metadata.return_value = flakiness_metadata # Flake object for s2_t1 exists. flake1 = Flake.Create( luci_project=luci_project, normalized_step_name='step1', normalized_test_name='s1_t1', test_label_name='s1_t1') flake1.put() detect_flake_occurrences.ProcessBuildForFlakes( detect_flake_occurrences.DetectFlakesFromFlakyCQBuildParam( build_id=build_id, flake_type_desc=FLAKE_TYPE_DESCRIPTIONS[flake_type_enum])) flake1_occurrence_num = FlakeOccurrence.query(ancestor=flake1.key).count() self.assertEqual(1, flake1_occurrence_num) flake2 = Flake.Get(luci_project, 'step1', 's1_t2') self.assertIsNotNone(flake2)
def setUp(self): super(CompileFailureRerunAnalysisTest, self).setUp() self.build_id = 8000000000123 self.build_number = 123 self.builder = BuilderID(project='chromium', bucket='ci', builder='linux-rel') self.build = Build(id=self.build_id, builder=self.builder, number=self.build_number, status=common_pb2.FAILURE) self.build.input.gitiles_commit.host = 'gitiles.host.com' self.build.input.gitiles_commit.project = 'project/name' self.build.input.gitiles_commit.ref = 'ref/heads/master' self.build.input.gitiles_commit.id = 'git_sha_123' self.build.create_time.FromDatetime(datetime(2019, 4, 9)) self.build.start_time.FromDatetime(datetime(2019, 4, 9, 0, 1)) self.build.end_time.FromDatetime(datetime(2019, 4, 9, 1)) self.context = Context(luci_project_name='chromium', gitiles_host='gitiles.host.com', gitiles_project='project/name', gitiles_ref='ref/heads/master', gitiles_id='git_sha') build_entity = LuciFailedBuild.Create( luci_project=self.context.luci_project_name, luci_bucket=self.build.builder.bucket, luci_builder=self.build.builder.builder, build_id=self.build_id, legacy_build_number=self.build_number, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, gitiles_id=self.context.gitiles_id, commit_position=6000005, status=20, create_time=datetime(2019, 3, 28), start_time=datetime(2019, 3, 28, 0, 1), end_time=datetime(2019, 3, 28, 1), build_failure_type=StepTypeEnum.COMPILE) build_entity.put() self.compile_failure = CompileFailure.Create( failed_build_key=build_entity.key, step_ui_name='compile', output_targets=['a.o'], first_failed_build_id=self.build_id, failure_group_build_id=None) self.compile_failure.put() self.analysis = CompileFailureAnalysis.Create( luci_project=self.context.luci_project_name, luci_bucket=self.build.builder.bucket, luci_builder=self.build.builder.builder, build_id=self.build_id, gitiles_host=self.context.gitiles_host, gitiles_project=self.context.gitiles_project, gitiles_ref=self.context.gitiles_ref, last_passed_gitiles_id='left_sha', last_passed_commit_position=6000000, first_failed_gitiles_id=self.context.gitiles_id, first_failed_commit_position=6000005, rerun_builder_id='chromium/findit/findit-variables', compile_failure_keys=[self.compile_failure.key]) self.analysis.Save()