def _GetAllTryJobs(self, failure_result_map): """Returns all try jobs related to one build. Args: A dict to map each step/test with the key to the build when it failed the first time. { 'step1': 'm/b/1', 'step2': { 'test1': 'm/b/1', 'test2': 'm/b/2' } } Returns: A dict of try jobs like below: { 'm/b/1': WfTryJob( key=Key('WfBuild', 'm/b/1'),...) ... } """ if not failure_result_map: return {} try_jobs = {} for step_map in failure_result_map.values(): if isinstance(step_map, basestring): try_jobs[step_map] = WfTryJob.Get(*step_map.split('/')) else: for task_key in step_map.values(): if not try_jobs.get(task_key): try_jobs[task_key] = WfTryJob.Get(*task_key.split('/')) return try_jobs
def testIdentifyCulpritForCompileTryJobNoCulprit(self): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '1' self._CreateEntities(master_name, builder_name, build_number, try_job_id) analysis = WfAnalysis.Create(master_name, builder_name, build_number) analysis.put() self.MockPipeline(RevertAndNotifyCulpritPipeline, None, expected_args=[master_name, builder_name, build_number, None, [], failure_type.COMPILE]) pipeline = IdentifyTryJobCulpritPipeline( master_name, builder_name, build_number, failure_type.COMPILE, '1', None) pipeline.start() self.execute_queued_tasks() try_job = WfTryJob.Get(master_name, builder_name, build_number) try_job_data = WfTryJobData.Get(try_job_id) self.assertEqual(analysis_status.COMPLETED, try_job.status) self.assertEqual([], try_job.compile_results) self.assertIsNone(try_job_data.culprits) self.assertIsNone(analysis.result_status) self.assertIsNone(analysis.suspected_cls)
def ScheduleCompileTryJob(parameters, runner_id): master_name, builder_name, build_number = (parameters.build_key.GetParts()) properties = GetBuildProperties(parameters) tryserver_mastername, tryserver_buildername = try_job_service.GetTrybot() build_id, error = try_job_service.TriggerTryJob( master_name, builder_name, tryserver_mastername, tryserver_buildername, properties, failure_type.GetDescriptionForFailureType(failure_type.COMPILE), parameters.cache_name, parameters.dimensions, runner_id) if error: raise exceptions.RetryException(error.reason, error.message) try_job = WfTryJob.Get(master_name, builder_name, build_number) try_job.compile_results.append({'try_job_id': build_id}) try_job.try_job_ids.append(build_id) try_job.put() try_job = try_job_service.UpdateTryJob(master_name, builder_name, build_number, build_id, failure_type.COMPILE) # Create a corresponding WfTryJobData entity to capture as much metadata as # early as possible. try_job_service.CreateTryJobData(build_id, try_job.key, bool(parameters.compile_targets), bool(parameters.suspected_revisions), failure_type.COMPILE, runner_id=runner_id) return build_id
def CreateTestFailureAnalysisCompletionEvent(analysis): """Transforms a test failure analysis into an event proto. Args: analysis (WfAnalysis): The analysis to be transformed. Returns: ([TestAnalysisCompletionEvent]) Proto used to report to BQ table. """ events = [] for step in analysis.failure_info.get('failed_steps', {}): for test in analysis.failure_info['failed_steps'][step].get( 'tests') or {}: if analysis.flaky_tests and test in analysis.flaky_tests.get( step, []): # The test is flaky, should report it in flake analysis. continue # If the failure result mapping isn't there, then bailout since it # contains required information. if (not analysis.failure_result_map or not analysis.failure_result_map.get(step) or not analysis.failure_result_map[step].get(test)): continue event = TestAnalysisCompletionEvent() event.flake = False _ExtractGeneralAnalysisInfo(analysis, event) event.analysis_info.step_name = step event.test_name = test # Extract master/builder/build_number from failure_result_Map. master, builder, build_number = ( analysis.failure_result_map[step][test].split('/')) event.analysis_info.culprit_build_number = int(build_number) # Culprit. try_job = WfTryJob.Get(master, builder, build_number) if (try_job and try_job.test_results and try_job.test_results[-1].get('culprit') and try_job.test_results[-1]['culprit'].get(step) and try_job.test_results[-1]['culprit'][step].get('tests') and try_job.test_results[-1]['culprit'][step]['tests'].get( test)): _ExtractCulpritForDict( try_job.test_results[-1]['culprit'][step]['tests'][test], event) event = _ExtractSuspectsForWfAnalysis(analysis, event) # Outcomes. _SetOutcomesForEvent(event) # Actions. _SetActionsForEvent(event) events.append(event) return events
def ReviveOrCreateTryJobEntity(master_name, builder_name, build_number, force_try_job): """Checks try job entity to further determine if need a new try job. * If there is an entity for a running or completed try job, no need for new job. * If there is an entity for a failed try job, revive the entity and start a new job. * If there is no entity, create one. Returns: A bool to indicate if a try job entity is revived or created. The try job entities' key. """ try_job_entity_revived_or_created = True try_job = WfTryJob.Get(master_name, builder_name, build_number) if try_job: if try_job.failed or force_try_job: try_job.status = analysis_status.PENDING try_job.put() else: try_job_entity_revived_or_created = False else: try_job = WfTryJob.Create(master_name, builder_name, build_number) try_job.put() return try_job_entity_revived_or_created, try_job.key.urlsafe()
def testReturnNoneIfNoTryJob(self): master_name = 'm' builder_name = 'b' build_number = 8 WfTryJob.Create(master_name, builder_name, build_number).put() self.MockGeneratorPipeline( pipeline_class=RevertAndNotifyTestCulpritPipeline, expected_input=CulpritActionParameters( build_key=BuildKey(master_name=master_name, builder_name=builder_name, build_number=build_number), culprits=DictOfBasestring(), heuristic_cls=ListOfBasestring(), failure_to_culprit_map=None), mocked_output=False) parameters = IdentifyTestTryJobCulpritParameters(build_key=BuildKey( master_name=master_name, builder_name=builder_name, build_number=build_number), result=None) pipeline = IdentifyTestTryJobCulpritPipeline(parameters) pipeline.start() self.execute_queued_tasks() try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(try_job.test_results, []) self.assertEqual(try_job.status, analysis_status.COMPLETED)
def testIdentifyCulpritForFlakyCompile(self): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '1' compile_result = { 'report': { 'result': { 'rev1': 'failed', 'rev2': 'failed' }, 'metadata': { 'sub_ranges': [ [ None, 'rev2' ] ] } }, 'url': 'url', 'try_job_id': try_job_id, } self._CreateEntities(master_name, builder_name, build_number, try_job_id) analysis = WfAnalysis.Create(master_name, builder_name, build_number) analysis.result = { 'failures': [ { 'step_name': 'compile', 'suspected_cls': [] } ] } analysis.put() self.MockPipeline(RevertAndNotifyCulpritPipeline, None, expected_args=[master_name, builder_name, build_number, {}, [], failure_type.COMPILE]) pipeline = IdentifyTryJobCulpritPipeline( master_name, builder_name, build_number, failure_type.COMPILE, '1', compile_result) pipeline.start() self.execute_queued_tasks() try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(analysis_status.COMPLETED, try_job.status) try_job_data = WfTryJobData.Get(try_job_id) self.assertIsNone(try_job_data.culprits) analysis = WfAnalysis.Get(master_name, builder_name, build_number) self.assertEqual(result_status.FLAKY, analysis.result_status) self.assertEqual([], analysis.suspected_cls)
def UpdateTryJobResult(parameters, culprits): master_name, builder_name, build_number = parameters.build_key.GetParts() try_job = WfTryJob.Get(master_name, builder_name, build_number) new_result = parameters.result.ToSerializable( ) if parameters.result else {} try_job_id = parameters.result.try_job_id if parameters.result else None if culprits: try_job_service.UpdateTryJobResult(try_job.compile_results, new_result, try_job_id) try_job.status = analysis_status.COMPLETED try_job.put()
def UpdateTryJob(master_name, builder_name, build_number, build_id, try_job_type): try_job = WfTryJob.Get(master_name, builder_name, build_number) if try_job_type == failure_type.COMPILE: try_job.compile_results.append({'try_job_id': build_id}) else: try_job.test_results.append({'try_job_id': build_id}) try_job.try_job_ids.append(build_id) try_job.put() return try_job
def testSuccessfullyScheduleNewTryJobForCompileWithSuspectedRevisions( self, mock_module): master_name = 'm' builder_name = 'b' build_number = 223 good_revision = 'rev1' bad_revision = 'rev2' build_id = '1' url = 'url' build = WfBuild.Create(master_name, builder_name, build_number) build.data = { 'properties': { 'parent_mastername': 'pm', 'parent_buildername': 'pb' } } build.put() response = { 'build': { 'id': build_id, 'url': url, 'status': 'SCHEDULED', } } results = [(None, buildbucket_client.BuildbucketBuild(response['build']))] mock_module.TriggerTryJobs.return_value = results WfTryJob.Create(master_name, builder_name, build_number).put() try_job_pipeline = ScheduleCompileTryJobPipeline() try_job_id = try_job_pipeline.run(master_name, builder_name, build_number, good_revision, bad_revision, failure_type.COMPILE, None, ['r5'], None, None) try_job = WfTryJob.Get(master_name, builder_name, build_number) try_job_data = WfTryJobData.Get(build_id) expected_try_job_id = '1' self.assertEqual(expected_try_job_id, try_job_id) self.assertEqual(expected_try_job_id, try_job.compile_results[-1]['try_job_id']) self.assertTrue(expected_try_job_id in try_job.try_job_ids) self.assertIsNotNone(try_job_data) self.assertEqual(try_job_data.master_name, master_name) self.assertEqual(try_job_data.builder_name, builder_name) self.assertEqual(try_job_data.build_number, build_number) self.assertEqual( try_job_data.try_job_type, failure_type.GetDescriptionForFailureType(failure_type.COMPILE)) self.assertFalse(try_job_data.has_compile_targets) self.assertTrue(try_job_data.has_heuristic_results)
def testGet(self): master_name = 'm' builder_name = 'b' build_number = 123 try_job_id = 'try_job_id' try_job_before = WfTryJob.Create(master_name, builder_name, build_number) try_job_before.try_job_ids = [try_job_id] try_job_before.put() try_job_after = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual([try_job_id], try_job_after.try_job_ids)
def run(self, master_name, builder_name, build_number, good_revision, bad_revision, try_job_type, suspected_revisions, cache_name, dimensions, *task_results): """ Args: master_name (str): the master name of a build. builder_name (str): the builder name of a build. build_number (int): the build number of a build. good_revision (str): the revision of the last passed build. bad__revision (str): the revision of the first failed build. try_job_type (int): type of the try job: TEST in this case. suspected_revisions (list): a list of suspected revisions from heuristic. cache_name (str): A string to identify separate directories for different waterfall bots on the trybots. dimensions (list): A list of strings in the format ["key1:value1", "key2:value2"]. task_results (list): a list of reliable failed tests. Returns: build_id (str): id of the triggered try job. """ properties = self._GetBuildProperties(master_name, builder_name, build_number, good_revision, bad_revision, try_job_type, suspected_revisions) targeted_tests = _GetTargetedTests(dict(task_results)) if not targeted_tests: # pragma: no cover logging.info('All tests are flaky, no try job will be triggered.') return additional_parameters = {'tests': targeted_tests} build_id = self._TriggerTryJob( master_name, builder_name, properties, additional_parameters, failure_type.GetDescriptionForFailureType(failure_type.TEST), cache_name, dimensions) try_job = WfTryJob.Get(master_name, builder_name, build_number) try_job.test_results.append({'try_job_id': build_id}) try_job.try_job_ids.append(build_id) try_job.put() # Create a corresponding WfTryJobData entity to capture as much metadata as # early as possible. self._CreateTryJobData(build_id, try_job.key, bool(suspected_revisions)) return build_id
def main(): start = datetime(2017, 12, 1, 0, 0, 0) cursor = None more = True groups_with_different_results = defaultdict(list) groups = defaultdict(list) while more: analyses, cursor, more = WfAnalysis.query( WfAnalysis.build_start_time >= start).fetch_page( 100, start_cursor=cursor) for analysis in analyses: if (analysis.status != analysis_status.COMPLETED or not analysis.failure_group_key or analysis.failure_type != failure_type.COMPILE): continue group_key = '/'.join(str(x) for x in analysis.failure_group_key) culprit = None try_job = WfTryJob.Get(*analysis.key.pairs()[0][1].split('/')) if try_job and try_job.compile_results: culprit = try_job.compile_results[-1].get('culprit') same_result = False for item in groups[group_key]: if (item['culprit'] != culprit or item['suspects'] != analysis.suspected_cls): continue same_result = True item['builds'].append(analysis.key.pairs()[0][1]) break if same_result: continue new_result = { 'suspects': analysis.suspected_cls, 'culprit': culprit, 'builds': [analysis.key.pairs()[0][1]] } groups[group_key].append(new_result) for key, item in groups.iteritems(): if len(item) > 1: groups_with_different_results[key] = item _DisplayResults(groups_with_different_results, groups)
def testUpdateTryJobResultAnalyzing(self): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '3' try_job = WfTryJob.Create(master_name, builder_name, build_number) try_job.put() pipeline = MonitorTryJobPipeline() pipeline._UpdateTryJobResult( try_job.key.urlsafe(), failure_type.TEST, try_job_id, 'url', buildbucket_client.BuildbucketBuild.STARTED) try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(analysis_status.RUNNING, try_job.status)
def _ReviveOrCreateTryJobEntity(master_name, builder_name, build_number, force_try_job): try_job_entity_revived_or_created = True try_job = WfTryJob.Get(master_name, builder_name, build_number) if try_job: if try_job.failed or force_try_job: try_job.status = analysis_status.PENDING try_job.put() else: try_job_entity_revived_or_created = False else: try_job = WfTryJob.Create(master_name, builder_name, build_number) try_job.put() return try_job_entity_revived_or_created, try_job.key
def CreateCompileFailureAnalysisCompletionEvent(analysis): """Transforms a compile failure analysis to an event proto. Args: analysis (WfAnalysis): The analysis to be transformed. Returns: (CompileAnalysisCompletionEvent) Proto used to report to BQ table. """ event = CompileAnalysisCompletionEvent() _ExtractGeneralAnalysisInfo(analysis, event) event.analysis_info.step_name = 'compile' if (analysis.failure_info and analysis.failure_info.get('failed_steps') and analysis.failure_info['failed_steps'].get('compile') and analysis.failure_info['failed_steps']['compile'].get( 'first_failure')): event.analysis_info.culprit_build_number = ( analysis.failure_info['failed_steps']['compile']['first_failure']) try_job = WfTryJob.Get(analysis.master_name, analysis.builder_name, event.analysis_info.culprit_build_number) # Culprit. if (try_job and try_job.compile_results and try_job.compile_results[-1].get('culprit') and try_job.compile_results[-1]['culprit'].get('compile')): _ExtractCulpritForDict( try_job.compile_results[-1]['culprit']['compile'], event) event = _ExtractSuspectsForWfAnalysis(analysis, event) if (analysis.signals and analysis.signals.get('compile') and analysis.signals['compile'].get('failed_edges')): # Use a set to avoid adding duplicates here. rules = set() for edge in analysis.signals['compile']['failed_edges']: rules.add(edge['rule']) event.failed_build_rules.extend(rules) # Outcomes. _SetOutcomesForEvent(event) # Actions. _SetActionsForEvent(event) return event
def testSuccessfullyScheduleNewTryJobForTest(self, mock_module): master_name = 'm' builder_name = 'b' build_number = 223 good_revision = 'rev1' bad_revision = 'rev2' targeted_tests = ['a on platform', ['a', ['test1', 'test2']]] build_id = '1' build = WfBuild.Create(master_name, builder_name, build_number) build.data = {'properties': {'parent_mastername': 'pm', 'parent_buildername': 'pb'}} build.put() response = { 'build': { 'id': build_id, 'url': 'url', 'status': 'SCHEDULED', } } results = [(None, buildbucket_client.BuildbucketBuild(response['build']))] mock_module.TriggerTryJobs.return_value = results WfTryJob.Create(master_name, builder_name, build_number).put() try_job_pipeline = ScheduleTestTryJobPipeline() try_job_id = try_job_pipeline.run( master_name, builder_name, build_number, good_revision, bad_revision, failure_type.TEST, None, None, None, targeted_tests) try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(try_job_id, build_id) self.assertEqual(try_job.test_results[-1]['try_job_id'], build_id) try_job_data = WfTryJobData.Get(try_job_id) self.assertIsNotNone(try_job_data) self.assertEqual(try_job_data.master_name, master_name) self.assertEqual(try_job_data.builder_name, builder_name) self.assertEqual(try_job_data.build_number, build_number) self.assertEqual( try_job_data.try_job_type, failure_type.GetDescriptionForFailureType(failure_type.TEST)) self.assertFalse(try_job_data.has_compile_targets) self.assertFalse(try_job_data.has_heuristic_results)
def run(self, master_name, builder_name, build_number, good_revision, bad_revision, try_job_type, compile_targets, suspected_revisions, cache_name, dimensions): """ Args: master_name (str): the master name of a build. builder_name (str): the builder name of a build. build_number (int): the build number of a build. good_revision (str): the revision of the last passed build. bad__revision (str): the revision of the first failed build. try_job_type (int): type of the try job: COMPILE in this case. compile_targets (list): a list of failed output nodes. suspected_revisions (list): a list of suspected revisions from heuristic. cache_name (str): A string to identify separate directories for different waterfall bots on the trybots. dimensions (list): A list of strings in the format ["key1:value1", "key2:value2"]. Returns: build_id (str): id of the triggered try job. """ properties = self._GetBuildProperties(master_name, builder_name, build_number, good_revision, bad_revision, try_job_type, suspected_revisions) additional_parameters = {'compile_targets': compile_targets} build_id = self._TriggerTryJob( master_name, builder_name, properties, additional_parameters, failure_type.GetDescriptionForFailureType(failure_type.COMPILE), cache_name, dimensions) try_job = WfTryJob.Get(master_name, builder_name, build_number) try_job.compile_results.append({'try_job_id': build_id}) try_job.try_job_ids.append(build_id) try_job.put() # Create a corresponding WfTryJobData entity to capture as much metadata as # early as possible. self._CreateTryJobData(build_id, try_job.key, bool(compile_targets), bool(suspected_revisions)) return build_id
def UpdateTryJobResult(parameters, culprits): """ Updates try job result with culprit info. Args: parameters (IdentifyTestTryJobCulpritParameters): Parameters to identify culprit from try job result. culprits (dict): A dict of culprits info: revision, repo_name, commit_position and url. """ master_name, builder_name, build_number = (parameters.build_key.GetParts()) try_job = WfTryJob.Get(master_name, builder_name, build_number) new_result = parameters.result.ToSerializable( ) if parameters.result else {} try_job_id = parameters.result.try_job_id if parameters.result else None if culprits: try_job_service.UpdateTryJobResult(try_job.test_results, new_result, try_job_id) try_job.status = analysis_status.COMPLETED try_job.put()
def testReturnNoneIfNoTryJob(self): master_name = 'm' builder_name = 'b' build_number = 8 WfTryJob.Create(master_name, builder_name, build_number).put() self.MockPipeline(RevertAndNotifyCulpritPipeline, None, expected_args=[master_name, builder_name, build_number, None, [], failure_type.TEST]) pipeline = IdentifyTryJobCulpritPipeline( master_name, builder_name, build_number, failure_type.TEST, None, None) pipeline.start() self.execute_queued_tasks() try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(try_job.test_results, []) self.assertEqual(try_job.status, analysis_status.COMPLETED)
def testIdentifyCulpritForCompileReturnNoneIfAllPassed(self): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '1' compile_result = { 'report': { 'result': { 'rev1': 'passed', 'rev2': 'passed' } }, 'url': 'url', 'try_job_id': try_job_id, } self._CreateEntities(master_name, builder_name, build_number, try_job_id) analysis = WfAnalysis.Create(master_name, builder_name, build_number) analysis.put() self.MockPipeline(RevertAndNotifyCulpritPipeline, None, expected_args=[master_name, builder_name, build_number, {}, [], failure_type.COMPILE]) pipeline = IdentifyTryJobCulpritPipeline( master_name, builder_name, build_number, failure_type.COMPILE, '1', compile_result) pipeline.start() self.execute_queued_tasks() try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(analysis_status.COMPLETED, try_job.status) try_job_data = WfTryJobData.Get(try_job_id) self.assertIsNone(try_job_data.culprits) self.assertIsNone(analysis.result_status) self.assertIsNone(analysis.suspected_cls)
def _PrepareTryJobDataForCompileFailure(analysis): try_job_data = {} if not (analysis.failure_result_map and # pragma: no branch. constants.COMPILE_STEP_NAME in analysis.failure_result_map): return try_job_data # pragma: no cover. referred_build_keys = build_util.GetBuildInfoFromId( analysis.failure_result_map[constants.COMPILE_STEP_NAME]) try_job = WfTryJob.Get(*referred_build_keys) if not try_job or not try_job.compile_results: return try_job_data # pragma: no cover. result = try_job.compile_results[-1] try_job_data['status'] = analysis_status.STATUS_TO_DESCRIPTION.get( try_job.status, 'unknown').lower() try_job_data['url'] = result.get('url') try_job_data['completed'] = try_job.completed try_job_data['failed'] = try_job.failed try_job_data['culprit'] = result.get('culprit', {}).get(constants.COMPILE_STEP_NAME) return try_job_data
def _GetTryJobResultForCompile(failure_result_map): try_job_key = failure_result_map['compile'] referred_build_keys = BaseBuildModel.GetBuildInfoFromBuildKey(try_job_key) culprit_info = defaultdict(lambda: defaultdict(list)) try_job = WfTryJob.Get(*referred_build_keys) if not try_job or try_job.test_results: return culprit_info try_job_result = ( try_job.compile_results[-1] if try_job.compile_results else None) compile_try_job = {'try_job_key': try_job_key, 'status': try_job.status} if try_job_result: if try_job_result.get('url'): compile_try_job['try_job_url'] = try_job_result['url'] compile_try_job['try_job_build_number'] = ( _GetTryJobBuildNumber(try_job_result)) if try_job_result.get('culprit', {}).get('compile'): compile_try_job['culprit'] = try_job_result['culprit']['compile'] culprit_info['compile']['try_jobs'].append(compile_try_job) return culprit_info
while more: analyses, cursor, more = WfAnalysis.query( ndb.AND(WfAnalysis.build_start_time >= start, WfAnalysis.build_start_time < end)).fetch_page( 100, start_cursor=cursor) for analysis in analyses: if not analysis.completed or not analysis.result: continue build_key = analysis.key.pairs()[0][1] master, builder_name, build_number = ( BaseBuildModel.GetBuildInfoFromBuildKey(build_key)) build_number = int(build_number) try_job = WfTryJob.Get(master, builder_name, build_number) for failure in analysis.result.get('failures', {}): result = None step_name = failure['step_name'] culprits = _GetTestTryJobCulprit(try_job, step_name) if failure.get('tests'): for test in failure['tests']: if test['first_failure'] != build_number: # Not first time failure. continue test_name = test['test_name'] result = Result(step_name, test_name)
def _GetCulpritInfoForTryJobResultForTest(try_job_key, culprits_info): referred_build_keys = BaseBuildModel.GetBuildInfoFromBuildKey(try_job_key) try_job = WfTryJob.Get(*referred_build_keys) if not try_job or try_job.compile_results: return try_job_result = try_job.test_results[-1] if try_job.test_results else None for step_try_jobs in culprits_info.values(): # If try job found different culprits for each test, split tests by culprit. additional_tests_culprit_info = [] for try_job_info in step_try_jobs['try_jobs']: if (try_job_key != try_job_info['try_job_key'] or try_job_info.get('status')): # Conditions that try_job_info has status are: # If there is no swarming task, there won't be try job; # If the swarming task is not completed yet, there won't be try job yet; # If there are flaky tests found, those tests will be marked as flaky, # and no try job for them will be triggered. continue try_job_info['status'] = try_job.status if try_job_result: # Needs to use ref_name to match step_name in try job. ref_name = try_job_info['ref_name'] # Saves try job information. if try_job_result.get('url'): # pragma: no cover try_job_info['try_job_url'] = try_job_result['url'] try_job_info['try_job_build_number'] = ( _GetTryJobBuildNumber(try_job_result)) if (try_job_result.get('culprit') and try_job_result['culprit'].get(ref_name)): # Saves try job culprits information. # Uses culprits to group tests. culprit_tests_map = _OrganizeTryJobResultByCulprits( try_job_result['culprit'][ref_name]) ungrouped_tests = try_job_info.get('tests', []) list_of_culprits = [] for culprit_info in culprit_tests_map.values(): failed_tests = culprit_info['failed_tests'] list_of_culprits.append(culprit_info) # Gets tests that haven't been grouped. ungrouped_tests = list(set(ungrouped_tests) ^ set(failed_tests)) if not ungrouped_tests: # All tests have been grouped. break index_start = 1 if ungrouped_tests: # There are tests don't have try job culprits. # Group these tests together. # Save them in current try_job_info. try_job_info['tests'] = ungrouped_tests try_job_info['culprit'] = {} # Saves all the tests that have culprits later. index_start = 0 else: # Saves the first culprit in current try_job_info. # Saves all the other culprits later. try_job_info['culprit'] = { 'revision': list_of_culprits[0]['revision'], 'commit_position': list_of_culprits[0]['commit_position'], 'review_url': list_of_culprits[0].get( 'url', list_of_culprits[0].get('review_url', None)) } try_job_info['tests'] = list_of_culprits[0]['failed_tests'] for n in xrange(index_start, len(list_of_culprits)): # Appends the rest of test groups to step_try_jobs['try_jobs']. iterate_culprit = list_of_culprits[n] tmp_try_job_info = copy.deepcopy(try_job_info) tmp_try_job_info['culprit'] = { 'revision': iterate_culprit['revision'], 'commit_position': iterate_culprit['commit_position'], 'review_url': iterate_culprit.get('url', iterate_culprit.get('review_url', None)) } tmp_try_job_info['tests'] = iterate_culprit['failed_tests'] additional_tests_culprit_info.append(tmp_try_job_info) if additional_tests_culprit_info: step_try_jobs['try_jobs'].extend(additional_tests_culprit_info)
def testGetTryJobsForTestSuccess(self, mock_buildbucket, mock_report): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '3' try_job = WfTryJob.Create(master_name, builder_name, build_number) try_job.test_results = [{ 'report': None, 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'try_job_id': try_job_id, }] try_job.status = analysis_status.RUNNING try_job.put() try_job_data = WfTryJobData.Create(try_job_id) try_job_data.try_job_key = try_job.key try_job_data.try_job_url = ( 'https://build.chromium.org/p/m/builders/b/builds/1234') try_job_data.put() data = [{ 'build': { 'id': '3', 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'status': 'STARTED' } }, { 'error': { 'reason': 'BUILD_NOT_FOUND', 'message': 'message', } }, { 'build': { 'id': '3', 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'status': 'STARTED' } }, { 'error': { 'reason': 'BUILD_NOT_FOUND', 'message': 'message', } }, { 'build': { 'id': '3', 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'status': 'COMPLETED', } }] report = { 'result': { 'rev1': { 'a_test': { 'status': 'passed', 'valid': True } }, 'rev2': { 'a_test': { 'status': 'failed', 'valid': True, 'failures': ['test1', 'test2'] } } } } get_tryjobs_responses = [ [(None, buildbucket_client.BuildbucketBuild(data[0]['build']))], [(buildbucket_client.BuildbucketError(data[1]['error']), None)], [(None, buildbucket_client.BuildbucketBuild(data[2]['build']))], [(buildbucket_client.BuildbucketError(data[3]['error']), None)], [(None, buildbucket_client.BuildbucketBuild(data[4]['build']))], ] mock_buildbucket.GetTryJobs.side_effect = get_tryjobs_responses mock_report.return_value = json.dumps(report) pipeline = MonitorTryJobPipeline() pipeline.start_test() pipeline.run(try_job.key.urlsafe(), failure_type.TEST, try_job_id) pipeline.run(try_job.key.urlsafe(), failure_type.TEST, try_job_id) # Since run() calls callback() immediately, we use -1. for _ in range(len(get_tryjobs_responses) - 1): pipeline.callback(callback_params=pipeline.last_params) # Reload from ID to get all internal properties in sync. pipeline = MonitorTryJobPipeline.from_id(pipeline.pipeline_id) pipeline.finalized() test_result = pipeline.outputs.default.value expected_test_result = { 'report': { 'result': { 'rev1': { 'a_test': { 'status': 'passed', 'valid': True } }, 'rev2': { 'a_test': { 'status': 'failed', 'valid': True, 'failures': ['test1', 'test2'] } } } }, 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'try_job_id': '3', } self.assertEqual(expected_test_result, test_result) try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(expected_test_result, try_job.test_results[-1]) self.assertEqual(analysis_status.RUNNING, try_job.status)
def testIdentifyCulpritForCompileTryJobSuccess(self): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '1' compile_result = { 'report': { 'result': { 'rev1': 'passed', 'rev2': 'failed' }, 'culprit': 'rev2' }, 'try_job_id': try_job_id, } self._CreateEntities(master_name, builder_name, build_number, try_job_id, try_job_status=analysis_status.RUNNING, compile_results=[compile_result]) analysis = WfAnalysis.Create(master_name, builder_name, build_number) analysis.put() expected_culprit = 'rev2' expected_suspected_cl = { 'revision': 'rev2', 'commit_position': 2, 'url': 'url_2', 'repo_name': 'chromium' } expected_compile_result = { 'report': { 'result': { 'rev1': 'passed', 'rev2': 'failed' }, 'culprit': 'rev2' }, 'try_job_id': try_job_id, 'culprit': { 'compile': expected_suspected_cl } } expected_analysis_suspected_cls = [{ 'revision': 'rev2', 'commit_position': 2, 'url': 'url_2', 'repo_name': 'chromium', 'failures': {'compile': []}, 'top_score': None }] self.MockPipeline(RevertAndNotifyCulpritPipeline, None, expected_args=[master_name, builder_name, build_number, {expected_culprit: expected_suspected_cl}, [], failure_type.COMPILE]) pipeline = IdentifyTryJobCulpritPipeline( master_name, builder_name, build_number, failure_type.COMPILE, '1', compile_result) pipeline.start() self.execute_queued_tasks() try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(expected_compile_result, try_job.compile_results[-1]) self.assertEqual(analysis_status.COMPLETED, try_job.status) try_job_data = WfTryJobData.Get(try_job_id) analysis = WfAnalysis.Get(master_name, builder_name, build_number) self.assertEqual({'compile': expected_culprit}, try_job_data.culprits) self.assertEqual(analysis.result_status, result_status.FOUND_UNTRIAGED) self.assertEqual(analysis.suspected_cls, expected_analysis_suspected_cls)
def testGetTryJobsForCompileSuccessBackwardCompatibleCallback( self, mock_buildbucket, mock_report): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '1' regression_range_size = 2 try_job = WfTryJob.Create(master_name, builder_name, build_number) try_job_data = WfTryJobData.Create(try_job_id) try_job_data.try_job_key = try_job.key try_job_data.try_job_url = ( 'https://build.chromium.org/p/m/builders/b/builds/1234') try_job_data.put() try_job.compile_results = [{ 'report': None, 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'try_job_id': '1', }] try_job.status = analysis_status.RUNNING try_job.put() report = { 'result': { 'rev1': 'passed', 'rev2': 'failed' }, 'metadata': { 'regression_range_size': 2 } } build_response = { 'id': '1', 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'status': 'COMPLETED', 'completed_ts': '1454367574000000', 'created_ts': '1454367570000000', 'updated_ts': '1454367574000000', } mock_buildbucket.GetTryJobs.return_value = [ (None, buildbucket_client.BuildbucketBuild(build_response)) ] mock_report.return_value = json.dumps(report) pipeline = MonitorTryJobPipeline(try_job.key.urlsafe(), failure_type.COMPILE, try_job_id) pipeline.start_test() pipeline.run(try_job.key.urlsafe(), failure_type.COMPILE, try_job_id) pipeline.callback(**pipeline.last_params) # Reload from ID to get all internal properties in sync. pipeline = MonitorTryJobPipeline.from_id(pipeline.pipeline_id) pipeline.finalized() compile_result = pipeline.outputs.default.value expected_compile_result = { 'report': { 'result': { 'rev1': 'passed', 'rev2': 'failed' }, 'metadata': { 'regression_range_size': regression_range_size } }, 'url': 'https://build.chromium.org/p/m/builders/b/builds/1234', 'try_job_id': '1', } self.assertEqual(expected_compile_result, compile_result) try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(expected_compile_result, try_job.compile_results[-1]) self.assertEqual(analysis_status.RUNNING, try_job.status) try_job_data = WfTryJobData.Get(try_job_id) self.assertEqual(try_job_data.regression_range_size, regression_range_size) self.assertIsInstance(try_job_data.start_time, datetime)
def testIdentifyCulpritForTestTryJobSuccess(self): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '1' test_result = { 'report': { 'result': { 'rev0': { 'a_test': { 'status': 'passed', 'valid': True, }, 'b_test': { 'status': 'failed', 'valid': True, 'failures': ['b_test1'] } }, 'rev1': { 'a_test': { 'status': 'failed', 'valid': True, 'failures': ['a_test1'] }, 'b_test': { 'status': 'failed', 'valid': True, 'failures': ['b_test1'] } }, 'rev2': { 'a_test': { 'status': 'failed', 'valid': True, 'failures': ['a_test1', 'a_test2'] }, 'b_test': { 'status': 'failed', 'valid': True, 'failures': ['b_test1'] } } }, 'culprits': { 'a_test': { 'a_test1': 'rev1', 'a_test2': 'rev2' }, }, 'flakes': { 'b_test': ['b_test1'] } }, 'url': 'url', 'try_job_id': try_job_id } self._CreateEntities(master_name, builder_name, build_number, try_job_id, try_job_status=analysis_status.RUNNING, test_results=[test_result]) analysis = WfAnalysis.Create(master_name, builder_name, build_number) analysis.put() a_test1_suspected_cl = { 'revision': 'rev1', 'commit_position': 1, 'url': 'url_1', 'repo_name': 'chromium' } a_test2_suspected_cl = { 'revision': 'rev2', 'commit_position': 2, 'url': 'url_2', 'repo_name': 'chromium' } expected_test_result = { 'report': { 'result': { 'rev0': { 'a_test': { 'status': 'passed', 'valid': True, }, 'b_test': { 'status': 'failed', 'valid': True, 'failures': ['b_test1'] } }, 'rev1': { 'a_test': { 'status': 'failed', 'valid': True, 'failures': ['a_test1'] }, 'b_test': { 'status': 'failed', 'valid': True, 'failures': ['b_test1'] } }, 'rev2': { 'a_test': { 'status': 'failed', 'valid': True, 'failures': ['a_test1', 'a_test2'] }, 'b_test': { 'status': 'failed', 'valid': True, 'failures': ['b_test1'] } } }, 'culprits': { 'a_test': { 'a_test1': 'rev1', 'a_test2': 'rev2' }, }, 'flakes': { 'b_test': ['b_test1'] } }, 'url': 'url', 'try_job_id': try_job_id, 'culprit': { 'a_test': { 'tests': { 'a_test1': a_test1_suspected_cl, 'a_test2': a_test2_suspected_cl } } } } expected_culprits = { 'rev1': a_test1_suspected_cl, 'rev2': a_test2_suspected_cl } self.MockPipeline(RevertAndNotifyCulpritPipeline, None, expected_args=[master_name, builder_name, build_number, expected_culprits, [], failure_type.TEST]) pipeline = IdentifyTryJobCulpritPipeline( master_name, builder_name, build_number, failure_type.TEST, '1', test_result) pipeline.start() self.execute_queued_tasks() try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(expected_test_result, try_job.test_results[-1]) self.assertEqual(analysis_status.COMPLETED, try_job.status) try_job_data = WfTryJobData.Get(try_job_id) analysis = WfAnalysis.Get(master_name, builder_name, build_number) expected_culprit_data = { 'a_test': { 'a_test1': 'rev1', 'a_test2': 'rev2', } } expected_cls = [ { 'revision': 'rev1', 'commit_position': 1, 'url': 'url_1', 'repo_name': 'chromium', 'failures': { 'a_test': ['a_test1'], 'b_test': ['b_test1'], }, 'top_score': None }, { 'revision': 'rev2', 'commit_position': 2, 'url': 'url_2', 'repo_name': 'chromium', 'failures': { 'a_test': ['a_test1', 'a_test2'], 'b_test': ['b_test1'], }, 'top_score': None } ] self.assertEqual(expected_culprit_data, try_job_data.culprits) self.assertEqual(analysis.result_status, result_status.FOUND_UNTRIAGED) self.assertEqual(analysis.suspected_cls, expected_cls)
def testGetTryJobsForCompileSuccessSwarming(self, mock_buildbucket, mock_report): master_name = 'm' builder_name = 'b' build_number = 1 try_job_id = '1' try_job_url = 'https://luci-milo.appspot.com/swarming/task/3595be5002f4bc10' regression_range_size = 2 try_job = WfTryJob.Create(master_name, builder_name, build_number) try_job_data = WfTryJobData.Create(try_job_id) try_job_data.try_job_key = try_job.key try_job_data.try_job_url = try_job_url try_job_data.put() try_job.compile_results = [{ 'report': None, 'url': try_job_url, 'try_job_id': '1', }] try_job.status = analysis_status.RUNNING try_job.put() build_response = { 'id': '1', 'url': try_job_url, 'status': 'COMPLETED', } report = { 'result': { 'rev1': 'passed', 'rev2': 'failed' }, 'metadata': { 'regression_range_size': 2 } } mock_buildbucket.GetTryJobs.return_value = [ (None, buildbucket_client.BuildbucketBuild(build_response)) ] mock_report.return_value = json.dumps(report) pipeline = MonitorTryJobPipeline() pipeline.start_test() pipeline.run(try_job.key.urlsafe(), failure_type.COMPILE, try_job_id) pipeline.callback(callback_params=pipeline.last_params) # Reload from ID to get all internal properties in sync. pipeline = MonitorTryJobPipeline.from_id(pipeline.pipeline_id) pipeline.finalized() compile_result = pipeline.outputs.default.value expected_compile_result = { 'report': { 'result': { 'rev1': 'passed', 'rev2': 'failed' }, 'metadata': { 'regression_range_size': regression_range_size } }, 'url': try_job_url, 'try_job_id': '1', } self.assertEqual(expected_compile_result, compile_result) try_job = WfTryJob.Get(master_name, builder_name, build_number) self.assertEqual(expected_compile_result, try_job.compile_results[-1]) self.assertEqual(analysis_status.RUNNING, try_job.status) try_job_data = WfTryJobData.Get(try_job_id) self.assertEqual(try_job_data.regression_range_size, regression_range_size)