def testOnSwarmingTaskCompleted(self, mock_mon, *_): master_name = 'm' builder_name = 'b' build_number = 13 step_name = 's' WfSwarmingTask.Create(master_name, builder_name, build_number, step_name).put() data = { 'state': constants.STATE_COMPLETED, 'created_ts': '2015-07-30T18:11:16.743220', 'started_ts': '2015-07-30T18:12:16.743220', 'completed_ts': '2015-07-30T18:15:16.743220' } test_swarming.OnSwarmingTaskCompleted(master_name, builder_name, build_number, step_name, data, 'output_json') swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) self.assertEqual({}, swarming_task.tests_statuses) self.assertEqual(analysis_status.COMPLETED, swarming_task.status) self.assertEqual(datetime.datetime(2015, 7, 30, 18, 11, 16, 743220), swarming_task.created_time) self.assertEqual(datetime.datetime(2015, 7, 30, 18, 12, 16, 743220), swarming_task.started_time) self.assertEqual(datetime.datetime(2015, 7, 30, 18, 15, 16, 743220), swarming_task.completed_time) mock_mon.assert_called_once_with(master_name, builder_name, build_number, step_name, analysis_status.COMPLETED, analysis_approach_type.SWARMING)
def testProcessSwarmingTaskResultPipelineTaskNotRunning(self): task = WfSwarmingTask.Create(self.master_name, self.builder_name, self.build_number, self.step_name) task.task_id = 'task_id2' task.put() pipeline = ProcessSwarmingTaskResultPipeline() pipeline.start_test() pipeline.run(self.master_name, self.builder_name, self.build_number, self.step_name) pipeline.callback(callback_params=pipeline.last_params) # Reload from ID to get all internal properties in sync. pipeline = ProcessSwarmingTaskResultPipeline.from_id( pipeline.pipeline_id) step_name, task_info = pipeline.outputs.default.value self.assertEqual(self.step_name, step_name) self.assertIsNone(task_info[0]) self.assertEqual([], task_info[1]) task = WfSwarmingTask.Get(self.master_name, self.builder_name, self.build_number, self.step_name) self.assertEqual(analysis_status.ERROR, task.status) self.assertEqual({}, task.tests_statuses) self.assertEqual({}, task.classified_tests)
def testProcessSwarmingTaskResultPipelineSerializedCallback(self, _): # End to end test. task = WfSwarmingTask.Create(self.master_name, self.builder_name, self.build_number, self.step_name) task.task_id = 'task_id1' task.put() pipeline = ProcessSwarmingTaskResultPipeline() pipeline.start_test() pipeline.run(self.master_name, self.builder_name, self.build_number, self.step_name) pipeline.callback(callback_params=json.dumps(pipeline.last_params)) # Reload from ID to get all internal properties in sync. pipeline = ProcessSwarmingTaskResultPipeline.from_id( pipeline.pipeline_id) pipeline.finalized() step_name, task_info = pipeline.outputs.default.value self.assertEqual(self.step_name, step_name) self.assertEqual('abc_tests', task_info[0]) self.assertEqual(_EXPECTED_CLASSIFIED_TESTS['reliable_tests'], task_info[1]) task = WfSwarmingTask.Get(self.master_name, self.builder_name, self.build_number, self.step_name) self.assertEqual(analysis_status.COMPLETED, task.status) self.assertEqual(_EXPECTED_TESTS_STATUS, task.tests_statuses) self.assertEqual(_EXPECTED_CLASSIFIED_TESTS, task.classified_tests) self.assertEqual(datetime.datetime(2016, 2, 10, 18, 32, 6, 538220), task.created_time) self.assertEqual(datetime.datetime(2016, 2, 10, 18, 32, 9, 90550), task.started_time) self.assertEqual(datetime.datetime(2016, 2, 10, 18, 33, 9), task.completed_time)
def testOnSwarmingTaskTimeout(self, mock_mon, _): master_name = 'm' builder_name = 'b' build_number = 16 step_name = 's' swarming_task = WfSwarmingTask.Create(master_name, builder_name, build_number, step_name) swarming_task.put() parameters = RunSwarmingTaskParameters(build_key=BuildKey( master_name=master_name, builder_name=builder_name, build_number=build_number), step_name=step_name, tests=['tests']) test_swarming.OnSwarmingTaskTimeout(parameters, 'task_id') swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) self.assertEqual(analysis_status.ERROR, swarming_task.status) self.assertEqual( { 'code': swarming_task_error.RUNNER_TIMEOUT, 'message': 'Runner to run swarming task timed out' }, swarming_task.error) mock_mon.assert_called_once_with(master_name, builder_name, build_number, step_name, analysis_status.ERROR, analysis_approach_type.SWARMING)
def testCollectSwarmingTaskResultsError(self, mock_monitoring, *_): master_name = 'm' builder_name = 'b' build_number = 15 params = CollectSwarmingTaskResultsInputs(build_key=BuildKey( master_name=master_name, builder_name=builder_name, build_number=build_number), build_completed=True) steps = ['step1', 'step2'] task1 = WfSwarmingTask.Create(master_name, builder_name, build_number, 'step1') task1.status = analysis_status.ERROR task1.put() task2 = WfSwarmingTask.Create(master_name, builder_name, build_number, 'step2') task2.status = analysis_status.COMPLETED task2.tests_statuses = { 'TestSuite1.test1': { 'total_run': 2, 'SUCCESS': 2 }, 'TestSuite1.test2': { 'total_run': 4, 'SUCCESS': 2, 'FAILURE': 2 }, 'TestSuite1.test3': { 'total_run': 6, 'FAILURE': 6 }, 'TestSuite1.test4': { 'total_run': 6, 'SKIPPED': 6 }, 'TestSuite1.test5': { 'total_run': 6, 'UNKNOWN': 6 } } task2.put() expected_result_json = { 'consistent_failures': { 'step2': ['TestSuite1.test3', 'TestSuite1.test4'] } } self.assertEqual( CollectSwarmingTaskResultsOutputs.FromSerializable( expected_result_json), test_swarming.GetConsistentFailuresWhenAllTasksComplete( params, steps)) mock_monitoring.assert_called_once_with('step2', 'step2', 'skip', 1)
def MockedSleep(*_): swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) self.assertEqual(analysis_status.PENDING, swarming_task.status) swarming_task.status = analysis_status.RUNNING swarming_task.task_id = 'task_id' swarming_task.put()
def testWaitingForTheTaskId(self): master_name = 'm' builder_name = 'b' build_number = 1 step_name = 's' swarming_task = WfSwarmingTask.Create(master_name, builder_name, build_number, step_name) swarming_task.status = analysis_status.PENDING swarming_task.put() def MockedSleep(*_): swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) self.assertEqual(analysis_status.PENDING, swarming_task.status) swarming_task.status = analysis_status.RUNNING swarming_task.task_id = 'task_id' swarming_task.put() self.mock(time, 'sleep', MockedSleep) self.assertEqual( 'task_id', swarmed_test_util.GetTaskIdFromSwarmingTaskEntity( swarming_task.key.urlsafe()))
def testGetTaskIdFromSwarmingTaskEntityNoTask(self): swarming_task = WfSwarmingTask.Create('m', 'b', 200, 's') swarming_task.put() key = swarming_task.key.urlsafe() swarming_task.key.delete() with self.assertRaises(Exception): swarmed_test_util.GetTaskIdFromSwarmingTaskEntity(key)
def testTriggerSwarmingTask(self, mock_ref, mock_create, mock_trigger, _): runner_id = 'runner_id' master_name = 'm' builder_name = 'b' build_number = 6 step_name = 's' tests = ['test'] WfSwarmingTask.Create(master_name, builder_name, build_number, step_name).put() parameters = RunSwarmingTaskParameters(build_key=BuildKey( master_name=master_name, builder_name=builder_name, build_number=build_number), step_name=step_name, tests=tests) new_request = SwarmingTaskRequest.FromSerializable( _SAMPLE_REQUEST_JSON) mock_create.return_value = new_request task_id = test_swarming.TriggerSwarmingTask(parameters, runner_id) self.assertEqual('new_task_id', task_id) mock_ref.assert_called_once_with(master_name, builder_name, build_number, step_name, None) mock_create.assert_called_once_with('runner_id', 'task_id', 'ref_request', master_name, builder_name, step_name, tests, 10) mock_trigger.assert_called_once_with('chromium-swarm.appspot.com', new_request, None)
def _GetHardTimeoutSeconds(master_name, builder_name, reference_build_number, step_name, iterations_to_rerun): flake_settings = waterfall_config.GetCheckFlakeSettings() flake_swarming_settings = flake_settings.get('swarming_rerun', {}) reference_task = WfSwarmingTask.Get(master_name, builder_name, reference_build_number, step_name) if _CanEstimateExecutionTimeFromReferenceSwarmingTask(reference_task): delta = reference_task.completed_time - reference_task.started_time execution_time = delta.total_seconds() number_of_tests = len(reference_task.tests_statuses) number_of_iterations = reference_task.parameters['iterations_to_rerun'] time_per_test_per_iteration = ( execution_time / (number_of_iterations * number_of_tests)) estimated_execution_time = (time_per_test_per_iteration * iterations_to_rerun) else: # Use default settings if the reference task is unavailable or malformed. estimated_execution_time = flake_swarming_settings.get( 'default_per_iteration_timeout_seconds', 60) * iterations_to_rerun # To account for variance and pending time, use a factor of 2x estimated # execution time. estimated_time_needed = estimated_execution_time * 2 return min(max(estimated_time_needed, _ONE_HOUR_IN_SECONDS), _MAX_TIMEOUT_SECONDS)
def _UpdateSwarmingTaskEntity(master_name, builder_name, build_number, step_name, status=None, task_id=None, error=None, classified_test_results=None, parameters=None, canonical_step_name=None, created_ts=None, started_ts=None, completed_ts=None): task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) assert task task.status = status or task.status task.task_id = task_id or task.task_id task.error = error.ToSerializable() if error else task.error task.classified_test_results = task.GetClassifiedTestResults( classified_test_results or {}) or task.classified_test_results task.parameters = task.parameters or {} task.parameters.update(parameters or {}) task.canonical_step_name = canonical_step_name or task.canonical_step_name task.created_time = task.created_time or time_util.DatetimeFromString( created_ts) task.started_time = task.started_time or time_util.DatetimeFromString( started_ts) task.completed_time = task.completed_time or time_util.DatetimeFromString( completed_ts) task.put()
def testTriggerFlakeAnalysesPipeline(self): master_name = 'm' builder_name = 'b' build_number = 2 step_name = 'a_tests' test_name = 'Unittest1.Subtest1' analysis = WfAnalysis.Create(master_name, builder_name, build_number) analysis.failure_result_map = { step_name: { test_name: '%s/%s/%s' % (master_name, builder_name, build_number) } } analysis.put() swarming_task = WfSwarmingTask.Create(master_name, builder_name, build_number, step_name) swarming_task.tests_statuses = {test_name: {'SUCCESS': 1}} swarming_task.put() with mock.patch.object(flake_analysis_service, 'ScheduleAnalysisForFlake') as ( mocked_ScheduleAnalysisForFlake): pipeline = TriggerFlakeAnalysesPipeline() pipeline.run(master_name, builder_name, build_number) mocked_ScheduleAnalysisForFlake.assert_called_once()
def testSwarmingPushMissingCallback(self, logging_mock): task = WfSwarmingTask.Create('m', 'b', 1, 'test') task.task_id = '12345' task.put() # This should not break, so that pubsub does not keep retrying. We'll only # log a message. self.test_app.post( '/pubsub/swarmingpush', params={ 'data': json.dumps({ 'message': { 'attributes': { 'auth_token': pubsub_callback.GetVerificationToken(), }, 'data': base64.b64encode( json.dumps({ 'task_id': '12345', 'userdata': json.dumps({ 'Message-Type': 'SwarmingTaskStatusChange', }), })), }, }), 'format': 'json', }) self.assertTrue(logging_mock.called)
def testSwarmingPush(self): task = WfSwarmingTask.Create('m', 'b', 1, 'test') task.task_id = '12345' task.callback_url = '/callback?pipeline_id=f9f89162ef32c7fb7' task.put() with mock.patch('google.appengine.api.taskqueue.add') as mock_queue: self.test_app.post( '/pubsub/swarmingpush', params={ 'data': json.dumps({ 'message': { 'attributes': { 'auth_token': pubsub_callback.GetVerificationToken(), }, 'data': base64.b64encode( json.dumps({ 'task_id': '12345', 'userdata': json.dumps({ 'Message-Type': 'SwarmingTaskStatusChange', }), })), }, }), 'format': 'json', }) mock_queue.assert_called_once()
def testGetStatusAndCulpritFromTryJobSwarmingTaskIsRunning(self): swarming_task = WfSwarmingTask.Create('m', 'b', 123, 'step') swarming_task.put() status, culprit = findit_api.FindItApi( )._GetStatusAndCulpritFromTryJob(None, swarming_task, None, 'step', None) self.assertEqual(status, findit_api._TryJobStatus.RUNNING) self.assertIsNone(culprit)
def testOnSwarmingTaskErrorShouldNotCompletePipeline(self, mock_mon): master_name = 'm' builder_name = 'b' build_number = 12 step_name = 's' WfSwarmingTask.Create(master_name, builder_name, build_number, step_name).put() error = {'code': 1, 'message': 'error'} test_swarming.OnSwarmingTaskError( master_name, builder_name, build_number, step_name, SwarmingTaskError.FromSerializable(error), False) swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) self.assertEqual(error, swarming_task.error) self.assertEqual(analysis_status.PENDING, swarming_task.status) self.assertFalse(mock_mon.called)
def testStepName(self): master_name = 'm' builder_name = 'b' build_number = 123 expected_step_name = 's' task = WfSwarmingTask.Create(master_name, builder_name, build_number, expected_step_name) self.assertEqual(expected_step_name, task.step_name)
def testGetTaskIdFromSwarmingTaskEntity(self): swarming_task = WfSwarmingTask.Create('m', 'b', 123, 's') swarming_task.task_id = 'task_id' swarming_task.put() self.assertEqual( 'task_id', swarmed_test_util.GetTaskIdFromSwarmingTaskEntity( swarming_task.key.urlsafe()))
def testNeedANewSwarmingTaskForce(self): swarming_task = WfSwarmingTask.Create('m', 'b', 2, 's') swarming_task.status = analysis_status.ERROR swarming_task.put() need, urlsafe_task_key = test_swarming.NeedANewSwarmingTask( 'm', 'b', 2, 's', True) self.assertTrue(need) swarming_task = ndb.Key(urlsafe=urlsafe_task_key).get() self.assertEqual(analysis_status.PENDING, swarming_task.status)
def _GetAllSwarmingTasks(self, failure_result_map): """Returns all swarming tasks related to one build. Args: A dict to map each step/test with the key to the build when it failed the first time. { 'step1': 'm/b/1', 'step2': { 'test1': 'm/b/1', 'test2': 'm/b/2' } } Returns: A dict of swarming tasks like below: { 'step1': { 'm/b/1': WfSwarmingTask( key=Key('WfBuild', 'm/b/1', 'WfSwarmingTask', 'step1'),...) }, ... } """ if not failure_result_map: return {} swarming_tasks = defaultdict(dict) for step_name, step_map in failure_result_map.iteritems(): if isinstance(step_map, basestring): swarming_tasks[step_name][step_map] = (WfSwarmingTask.Get( *BaseBuildModel.GetBuildInfoFromBuildKey(step_map), step_name=step_name)) else: for task_key in step_map.values(): if not swarming_tasks[step_name].get(task_key): swarming_tasks[step_name][task_key] = ( WfSwarmingTask.Get( *BaseBuildModel.GetBuildInfoFromBuildKey( task_key), step_name=step_name)) return swarming_tasks
def testCollectSwarmingTaskResultsTaskRunning(self, mock_monitoring, _): master_name = 'm' builder_name = 'b' build_number = 15 params = CollectSwarmingTaskResultsInputs(build_key=BuildKey( master_name=master_name, builder_name=builder_name, build_number=build_number), build_completed=True) steps = ['step1', 'step2'] task1 = WfSwarmingTask.Create(master_name, builder_name, build_number, 'step2') task1.status = analysis_status.COMPLETED task1.put() WfSwarmingTask.Create(master_name, builder_name, build_number, 'step1').put() self.assertIsNone( test_swarming.GetConsistentFailuresWhenAllTasksComplete( params, steps)) self.assertFalse(mock_monitoring.called)
def NeedANewSwarmingTask(master_name, builder_name, build_number, step_name, force): """Checks if a WfSwarmingTask for the given params exists, or creates it.""" swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) if not swarming_task: swarming_task = WfSwarmingTask.Create(master_name, builder_name, build_number, step_name) swarming_task.status = analysis_status.PENDING swarming_task.put() return True, swarming_task.key.urlsafe() if force: swarming_task.Reset() swarming_task.put() return True, swarming_task.key.urlsafe() # TODO(http://crbug.com/585676): Rerun the Swarming task if it runs into # unexpected infra errors. return False, swarming_task.key.urlsafe()
def testOnSwarmingTaskErrorShouldCompletePipeline(self, mock_mon): master_name = 'm' builder_name = 'b' build_number = 11 step_name = 's' WfSwarmingTask.Create(master_name, builder_name, build_number, step_name).put() error = {'code': 1, 'message': 'error'} self.assertFalse( test_swarming.OnSwarmingTaskError( master_name, builder_name, build_number, step_name, SwarmingTaskError.FromSerializable(error))) swarming_task = WfSwarmingTask.Get(master_name, builder_name, build_number, step_name) self.assertEqual(error, swarming_task.error) self.assertEqual(analysis_status.ERROR, swarming_task.status) mock_mon.assert_called_once_with(master_name, builder_name, build_number, step_name, analysis_status.ERROR, analysis_approach_type.SWARMING)
def testMonitorSwarmingTaskGetSwarmingTaskFailureLogError(self, _): task = WfSwarmingTask.Create(self.master_name, self.builder_name, self.build_number, self.step_name) task.task_id = 'task_id1' task.put() pipeline = ProcessSwarmingTaskResultPipeline() pipeline.start_test() pipeline.run(self.master_name, self.builder_name, self.build_number, self.step_name) self.assertEqual(analysis_status.ERROR, task.status) self.assertEqual(task.error, {'code': 1, 'message': 'error'})
def testGetSwarmingTask(self): master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' WfSwarmingTask.Create(master_name, builder_name, build_number, step_name).put() task = TriggerSwarmingTaskPipeline()._GetSwarmingTask( master_name, builder_name, build_number, step_name) self.assertEqual(master_name, task.master_name) self.assertEqual(builder_name, task.builder_name) self.assertEqual(build_number, task.build_number) self.assertEqual(step_name, task.step_name)
def testOnSwarmingTaskStateChangedError(self, *_): master_name = 'm' builder_name = 'b' build_number = 10 step_name = 's' WfSwarmingTask.Create(master_name, builder_name, build_number, step_name).put() parameters = RunSwarmingTaskParameters(build_key=BuildKey( master_name=master_name, builder_name=builder_name, build_number=build_number), step_name=step_name, tests=['test']) self.assertFalse( test_swarming.OnSwarmingTaskStateChanged(parameters, 'task_id'))
def testNoNewSwarmingTaskIsNeeded(self): master_name = 'm' builder_name = 'b' build_number = 1 step_name = 's' tests = ['a.b'] swarming_task = WfSwarmingTask.Create(master_name, builder_name, build_number, step_name) swarming_task.status = analysis_status.RUNNING swarming_task.task_id = 'task_id' swarming_task.put() pipeline = TriggerSwarmingTaskPipeline() task_id = pipeline.run(master_name, builder_name, build_number, step_name, tests) self.assertEqual('task_id', task_id)
def testMonitorSwarmingTaskWhereNoTaskOutputs(self): task = WfSwarmingTask.Create(self.master_name, self.builder_name, self.build_number, self.step_name) task.task_id = 'task_id4' task.put() pipeline = ProcessSwarmingTaskResultPipeline() pipeline.start_test() pipeline.run(self.master_name, self.builder_name, self.build_number, self.step_name) self.assertEqual(analysis_status.ERROR, task.status) self.assertEqual( task.error, { 'code': swarming_util.NO_TASK_OUTPUTS, 'message': 'outputs_ref is None' })
def run(self, master_name, builder_name, build_number): """Triggers flake analyses for flaky tests found by build failure analysis. Args: master_name (str): The master name. builder_name (str): The builder name. build_number (str): The build number. """ analysis = WfAnalysis.Get(master_name, builder_name, build_number) if not analysis or not analysis.failure_result_map: # pragma: no cover return for step in analysis.failure_result_map.iterkeys(): task = WfSwarmingTask.Get( master_name, builder_name, build_number, step) if not task: # pragma: no cover continue flaky_tests = task.classified_tests.get('flaky_tests', []) if not flaky_tests: # pragma: no cover continue # Trigger a master flake analysis on each detected flaky test. # TODO lijeffrey): rerun all tests once typical load is determined to be # within reasonable limits. For experimentation with automatic flakiness # checking, only run 1 test per anaysis to avoid excessive load on the # swarming server in case there are too many flaky tests per analysis for # now. test_name = flaky_tests[0] request = FlakeAnalysisRequest.Create(test_name, False, None) request.AddBuildStep( master_name, builder_name, build_number, step, time_util.GetUTCNow()) scheduled = flake_analysis_service.ScheduleAnalysisForFlake( request, '*****@*****.**', False, triggering_sources.FINDIT_PIPELINE) if scheduled: # pragma: no branch logging.info('%s/%s/%s has %s flaky tests.', master_name, builder_name, build_number, len(flaky_tests)) logging.info('A flake analysis has been triggered for %s', test_name)
def testMonitorSwarmingTaskWhereTaskFailed(self): task = WfSwarmingTask.Create(self.master_name, self.builder_name, self.build_number, self.step_name) task.task_id = 'task_id3' task.put() pipeline = ProcessSwarmingTaskResultPipeline() pipeline.start_test() pipeline.run(self.master_name, self.builder_name, self.build_number, self.step_name) self.assertEqual(analysis_status.ERROR, task.status) self.assertEqual( task.error, { 'code': swarming_util.TASK_FAILED, 'message': swarming_util.EXIT_CODE_DESCRIPTIONS[swarming_util.TASK_FAILED] })
html += cell_template % _FormatDigits(builder_report['error_rate']) html += '</table>' return html if __name__ == '__main__': # Set up the Remote API to use services on the live App Engine. remote_api.EnableRemoteApi(app_id='findit-for-me') START_DATE = datetime.datetime(2016, 2, 1) END_DATE = datetime.datetime(2016, 3, 7) wf_analysis_query = WfSwarmingTask.query( WfSwarmingTask.created_time >= START_DATE, WfSwarmingTask.created_time < END_DATE) data_list = wf_analysis_query.fetch() categorized_data_dict = _CategorizeSwarmingTaskData(data_list) final_report = _GetReport(categorized_data_dict, START_DATE, END_DATE) findit_tmp_dir = os.environ.get('TMP_DIR') if not findit_tmp_dir: findit_tmp_dir = os.getcwd() report_path = os.path.join(findit_tmp_dir, 'swarming_task_metadata_report.html') with open(report_path, 'w') as f: f.write(CreateHtmlPage(final_report, START_DATE, END_DATE))