def testPost_WithValidBugButNoComments(self): self.SetCurrentUserOAuth(testing_common.INTERNAL_USER) try_job.TryJob(bug_id=123456, status='started', bot='win_perf', results_data={}, config='config = {"command": "cmd"}', last_ran_timestamp=datetime.datetime(2017, 01, 01)).put() try_job.TryJob(bug_id=123456, status='failed', bot='android_bisect', results_data={ 'metric': 'foo' }, config='config = {"command": "cmd"}').put() try_job.TryJob(bug_id=99999, status='failed', bot='win_perf', results_data={ 'metric': 'foo' }, config='config = {"command": "cmd"}').put() response = self.Post('/api/bugs/123456') bug = self.GetJsonValue(response, 'bug') self.assertNotIn('comments', bug)
def testGet_FailsToUpdateBug_LogsErrorAndMovesOn(self, mock_logging_error): # Put a successful job and a failed job with partial results. # Note that AddBugComment is mocked to always returns false, which # simulates failing to post results to the issue tracker for all bugs. try_job.TryJob(bug_id=12345, rietveld_issue_id=200034, rietveld_patchset_id=1, status='started', bot='win_perf').put() try_job.TryJob(bug_id=54321, rietveld_issue_id=200039, rietveld_patchset_id=1, status='started', bot='win_perf').put() bug_data.Bug(id=12345).put() bug_data.Bug(id=54321).put() self.testapp.get('/update_bug_with_results') # Two errors should be logged. self.assertEqual(2, mock_logging_error.call_count) mock_logging_error.assert_called_with('Caught Exception %s: %s', 'BugUpdateFailure', mock.ANY) # The pending jobs should still be there. pending_jobs = try_job.TryJob.query().fetch() self.assertEqual(2, len(pending_jobs)) self.assertEqual('started', pending_jobs[0].status) self.assertEqual('started', pending_jobs[1].status)
def testGet_MergesBugIntoExistingBug(self, mock_update_bug): # When there exists a bug with the same revision (commit hash), # mark bug as duplicate and merge current issue into that. try_job.TryJob(bug_id=12345, rietveld_issue_id=200037, rietveld_patchset_id=1, status='started', bot='win_perf').put() try_job.TryJob(bug_id=54321, rietveld_issue_id=200037, rietveld_patchset_id=1, status='started', bot='win_perf').put() # Create bug. bug_data.Bug(id=12345).put() bug_data.Bug(id=54321).put() self.testapp.get('/update_bug_with_results') # Owners of CLs are not cc'ed for duplicate bugs and the issue should be # marked as duplicate. mock_update_bug.assert_called_with(mock.ANY, mock.ANY, cc_list=[], merge_issue='12345', labels=None, owner=None) pending_jobs = try_job.TryJob.query().fetch() self.assertEqual(0, len(pending_jobs)) # Add anomalies. test_keys = map(utils.TestKey, [ 'ChromiumGPU/linux-release/scrolling-benchmark/first_paint', 'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time' ]) anomaly.Anomaly(start_revision=9990, end_revision=9997, test=test_keys[0], median_before_anomaly=100, median_after_anomaly=200, sheriff=None, bug_id=12345).put() anomaly.Anomaly(start_revision=9990, end_revision=9996, test=test_keys[0], median_before_anomaly=100, median_after_anomaly=200, sheriff=None, bug_id=54321).put() # Map anomalies to base(dest_bug_id) bug. update_bug_with_results._MapAnomaliesToMergeIntoBug( dest_bug_id=12345, source_bug_id=54321) anomalies = anomaly.Anomaly.query( anomaly.Anomaly.bug_id == int(54321)).fetch() self.assertEqual(0, len(anomalies))
def testPost_WithValidBug_ShowsData(self, mock_oauth, mock_utils): self._SetGooglerOAuth(mock_oauth) mock_utils.return_value = True try_job.TryJob(bug_id=123456, status='started', bot='win_perf', results_data={}, config='config = {"command": "cmd"}', last_ran_timestamp=datetime.datetime(2017, 01, 01)).put() try_job.TryJob(bug_id=123456, status='failed', bot='android_bisect', results_data={ 'metric': 'foo' }, config='config = {"command": "cmd"}', last_ran_timestamp=datetime.datetime(2017, 01, 01)).put() try_job.TryJob(bug_id=99999, status='failed', bot='win_perf', results_data={ 'metric': 'foo' }, config='config = {"command": "cmd"}', last_ran_timestamp=datetime.datetime(2017, 01, 01)).put() response = self.testapp.post('/api/bugs/123456') bug = self.GetJsonValue(response, 'bug') self.assertEqual('The bug title', bug.get('summary')) self.assertEqual(2, len(bug.get('cc'))) self.assertEqual('*****@*****.**', bug.get('cc')[1]) self.assertEqual('Fixed', bug.get('status')) self.assertEqual('closed', bug.get('state')) self.assertEqual('*****@*****.**', bug.get('author')) self.assertEqual('*****@*****.**', bug.get('owner')) self.assertEqual('2017-06-28T01:26:53', bug.get('published')) self.assertEqual('2018-03-01T16:16:22', bug.get('updated')) self.assertEqual(2, len(bug.get('comments'))) self.assertEqual('Comment two', bug.get('comments')[1].get('content')) self.assertEqual('*****@*****.**', bug.get('comments')[1].get('author')) self.assertEqual(2, len(bug.get('legacy_bisects'))) self.assertEqual('started', bug.get('legacy_bisects')[0].get('status')) self.assertEqual('cmd', bug.get('legacy_bisects')[0].get('command')) self.assertEqual('2017-01-01T00:00:00', bug.get('legacy_bisects')[0].get('started_timestamp'))
def _MakeBisectFYITryJob(test_name, bisect_config): """Creates a TryJob entity with the bisect config. Args: test_name: Name of the test case. bisect_config: A dictionary of parameters for a bisect job. Returns: A TryJob entity, which has not yet been put in the datastore. Raises: NotBisectableError: A valid bisect config could not be created. """ bisect_bot = bisect_config.get('recipe_tester_name') if not bisect_bot: raise auto_bisect.NotBisectableError('Could not select a bisect bot.') config_python_string = utils.BisectConfigPythonString(bisect_config) bisect_job = try_job.TryJob(bot=bisect_bot, config=config_python_string, bug_id=bisect_config.get('bug_id', -1), master_name=bisect_config.get( 'master_name', 'ChromiumPerf'), job_type='bisect-fyi', job_name=test_name) return bisect_job
def testGet_PerfTryJob(self): try_job.TryJob(rietveld_issue_id=200034, rietveld_patchset_id=1, status='started', bot='win_perf', email='*****@*****.**', job_type='perf-try', config=_PERF_TEST_CONFIG).put() global _TEST_RECEIEVED_EMAIL_RESULTS _TEST_RECEIEVED_EMAIL_RESULTS = None self.testapp.get('/update_bug_with_results') results = _TEST_RECEIEVED_EMAIL_RESULTS self.assertEqual('Completed', results['status']) self.assertEqual(2, len(results['profiler_results'])) self.assertEqual(_PERF_LOG_EXPECTED_HTML_LINK, results['html_results']) self.assertEqual(_PERF_LOG_EXPECTED_TITLE_1, results['profiler_results'][0][0]) self.assertEqual(_PERF_LOG_EXPECTED_PROFILER_LINK1, results['profiler_results'][0][1]) self.assertEqual(_PERF_LOG_EXPECTED_TITLE_2, results['profiler_results'][1][0]) self.assertEqual(_PERF_LOG_EXPECTED_PROFILER_LINK2, results['profiler_results'][1][1]) self.assertEqual('win_perf_bisect', results['bisect_bot'])
def _PerformPerfTryStep(self, user): """Gathers the parameters required for a perf try job and starts the job.""" perf_config = _GetPerfTryConfig( bisect_bot=self.request.get('bisect_bot'), suite=self.request.get('suite'), good_revision=self.request.get('good_revision'), bad_revision=self.request.get('bad_revision'), rerun_option=self.request.get('rerun_option')) if 'error' in perf_config: return perf_config config_python_string = 'config = %s\n' % json.dumps( perf_config, sort_keys=True, indent=2, separators=(',', ': ')) perf_job = try_job.TryJob(bot=self.request.get('bisect_bot'), config=config_python_string, bug_id=-1, email=user.email(), job_type='perf-try') results = _PerformPerfTryJob(perf_job) if 'error' in results and perf_job.key: perf_job.key.delete() return results
def testPost_JobRunTooManyTimes_LogsMessage(self, mock_log_result): job_key = try_job.TryJob( bug_id=333, status='failed', last_ran_timestamp=datetime.datetime.now(), run_count=len(auto_bisect._BISECT_RESTART_PERIOD_DAYS) + 1).put() self.testapp.post('/auto_bisect') self.assertIsNone(job_key.get()) mock_log_result.assert_called_once_with(333, mock.ANY)
def testPost_FailedJobRunOnce_JobRestarted(self, mock_perform_bisect): try_job.TryJob(bug_id=222, status='failed', last_ran_timestamp=datetime.datetime.now(), run_count=1).put() self.testapp.post('/auto_bisect') mock_perform_bisect.assert_called_once_with( try_job.TryJob.query(try_job.TryJob.bug_id == 222).get())
def testPerformBisect_InvalidConfig_ReturnsError(self): bisect_job = try_job.TryJob(bot='foo', config='config = {}', master_name='ChromiumPerf', internal_only=False, job_type='bisect') self.assertEqual({'error': 'No "recipe_tester_name" given.'}, start_try_job.PerformBisect(bisect_job))
def testGet_WithStatsParameter_ListsTryJobs(self): now = datetime.datetime.now() try_job.TryJob( bug_id=222, status='failed', last_ran_timestamp=now, run_count=2).put() try_job.TryJob( bug_id=444, status='started', last_ran_timestamp=now, run_count=1).put() try_job.TryJob( bug_id=777, status='started', last_ran_timestamp=now, run_count=1).put() try_job.TryJob( bug_id=555, status=None, last_ran_timestamp=now, run_count=1).put() response = self.testapp.get('/auto_bisect?stats') self.assertIn('Failed jobs: 1', response.body) self.assertIn('Started jobs: 2', response.body)
def testGet_NegativeResult_StoresCommitHash(self): try_job.TryJob(bug_id=12345, rietveld_issue_id=200034, rietveld_patchset_id=1, status='started', bot='win_perf').put() self.testapp.get('/update_bug_with_results') self.assertIsNone(layered_cache.Get('commit_hash_a121212'))
def setUp(self): super(BadBisectHandlerTest, self).setUp() app = webapp2.WSGIApplication([('/bad_bisect', bad_bisect.BadBisectHandler)]) self.testapp = webtest.TestApp(app) testing_common.SetSheriffDomains(['chromium.org']) testing_common.SetIsInternalUser('*****@*****.**', True) self.SetCurrentUser('*****@*****.**') try_job.TryJob(id=1234).put()
def testGet_PostResult_WithoutBugEntity( self, mock_update_bug): job = try_job.TryJob(bug_id=12345, status='started', bot='win_perf', results_data=_SAMPLE_BISECT_RESULTS_JSON) job.put() self.testapp.get('/update_bug_with_results') mock_update_bug.assert_called_once_with( 12345, mock.ANY, cc_list=mock.ANY, merge_issue=mock.ANY, labels=mock.ANY, owner=mock.ANY)
def testPost_NoExceptionInPerformBisect_CustomMetricTicked( self, mock_tick, mock_perform_bisect): try_job.TryJob(bug_id=222, status='failed', last_ran_timestamp=datetime.datetime.now(), run_count=1).put() self.testapp.post('/auto_bisect') self.assertEqual(1, mock_perform_bisect.call_count) mock_tick.assert_called_once_with('RestartFailedBisectJobs')
def testPost_RunCount1_ExceptionInPerformBisect_CustomMetricNotTicked( self, mock_tick, mock_perform_bisect): mock_perform_bisect.side_effect = request_handler.InvalidInputError() try_job.TryJob(bug_id=222, status='failed', last_ran_timestamp=datetime.datetime.now(), run_count=1).put() self.testapp.post('/auto_bisect') self.assertEqual(0, mock_tick.call_count)
def testPost_RunCount2_ExceptionInPerformBisect_CustomMetricNotTicked( self, mock_tick, mock_perform_bisect): mock_perform_bisect.side_effect = Exception('Error') try_job.TryJob( bug_id=111, status='failed', last_ran_timestamp=datetime.datetime.now() - datetime.timedelta(days=8), run_count=2).put() self.testapp.post('/auto_bisect') self.assertEqual(0, mock_tick.call_count)
def testPost(self): job_key = try_job.TryJob(id=6789, rietveld_issue_id=200034).put() data_param = json.dumps(_SAMPLE_BISECT_RESULTS_JSON) self.testapp.post('/post_bisect_results', {'data': data_param}, extra_environ={'REMOTE_ADDR': _WHITELISTED_IP}) job = job_key.get() self.assertEqual(6789, job.results_data['try_job_id']) self.assertEqual('completed', job.results_data['status'])
def _MakeBisectTryJob(bug_id, test_anomaly, test): """Tries to automatically select parameters for a bisect job. Args: bug_id: A bug ID which some alerts are associated with. Returns: A TryJob entity, which has not yet been put in the datastore. Raises: NotBisectableError: A valid bisect config could not be created. """ good_revision = _GetRevisionForBisect(test_anomaly.start_revision - 1, test) bad_revision = _GetRevisionForBisect(test_anomaly.end_revision, test) if not can_bisect.IsValidRevisionForBisect(good_revision): raise NotBisectableError('Invalid "good" revision: %s.' % good_revision) if not can_bisect.IsValidRevisionForBisect(bad_revision): raise NotBisectableError('Invalid "bad" revision: %s.' % bad_revision) if test_anomaly.start_revision == test_anomaly.end_revision: raise NotBisectableError('Same "good"/"bad" revisions, bisect skipped') metric = start_try_job.GuessMetric(test.test_path) story_filter = start_try_job.GuessStoryFilter(test.test_path) bisect_bot = start_try_job.GuessBisectBot(test.master_name, test.bot_name) if not bisect_bot: raise NotBisectableError( 'Could not select a bisect bot: %s for (%s, %s)' % (bisect_bot, test.master_name, test.bot_name)) new_bisect_config = start_try_job.GetBisectConfig( bisect_bot=bisect_bot, master_name=test.master_name, suite=test.suite_name, metric=metric, story_filter=story_filter, good_revision=good_revision, bad_revision=bad_revision, repeat_count=10, max_time_minutes=20, bug_id=bug_id) if 'error' in new_bisect_config: raise NotBisectableError('Could not make a valid config.') config_python_string = utils.BisectConfigPythonString(new_bisect_config) bisect_job = try_job.TryJob(bot=bisect_bot, config=config_python_string, bug_id=bug_id, master_name=test.master_name, internal_only=test.internal_only, job_type='bisect') return bisect_job
def testGet_InProgressResult_BuildbucketFailure(self): sample_bisect_results = copy.deepcopy(_SAMPLE_BISECT_RESULTS_JSON) sample_bisect_results['status'] = 'started' job_key = try_job.TryJob(bug_id=12345, status='started', bot='win_perf', results_data=sample_bisect_results).put() self.testapp.get('/update_bug_with_results') job = job_key.get() self.assertEqual(job.results_data['status'], 'started')
def testValidateBuildbucketResponse_Started(self): job = try_job.TryJob(bug_id=12345, status='started', bot='win_perf') job.put() buildbucket_response_started = r"""{ "build": { "status": "STARTED", "id": "9043191319901995952" } }""" self.assertFalse(update_bug_with_results._ValidateBuildbucketResponse( json.loads(buildbucket_response_started), job))
def testGet(self): # Put succeeded, failed, and not yet finished jobs in the datastore. try_job.TryJob(bug_id=12345, rietveld_issue_id=200034, rietveld_patchset_id=1, status='started', bot='win_perf').put() try_job.TryJob(bug_id=54321, rietveld_issue_id=302304, rietveld_patchset_id=1, status='started', bot='win_perf').put() try_job.TryJob(bug_id=99999, rietveld_issue_id=100001, rietveld_patchset_id=1, status='started', bot='win_perf').put() try_job.TryJob(bug_id=77777, buildbucket_job_id='1234567', use_buildbucket=True, status='started', bot='win_perf').put() # Create bug. bug_data.Bug(id=12345).put() bug_data.Bug(id=54321).put() bug_data.Bug(id=99999).put() bug_data.Bug(id=77777).put() self.testapp.get('/update_bug_with_results') pending_jobs = try_job.TryJob.query().fetch() # Expects a failed and not yet finished bisect job to be in datastore. self.assertEqual(3, len(pending_jobs)) self.assertEqual(54321, pending_jobs[0].bug_id) self.assertEqual('failed', pending_jobs[0].status) self.assertEqual(99999, pending_jobs[1].bug_id) self.assertEqual(77777, pending_jobs[2].bug_id) self.assertEqual('started', pending_jobs[1].status) self.assertEqual('started', pending_jobs[2].status) self.assertEqual('bisect', pending_jobs[0].job_type) self.assertEqual('bisect', pending_jobs[1].job_type) self.assertEqual('bisect', pending_jobs[2].job_type)
def testPost_InProgress(self): job_key = try_job.TryJob(id=6790, rietveld_issue_id=200035, bug_id=10).put() data = copy.deepcopy(_SAMPLE_BISECT_RESULTS_JSON) data['status'] = 'started' data['try_job_id'] = 6790 data_param = json.dumps(data) self.testapp.post( '/post_bisect_results', {'data': data_param}, extra_environ={'REMOTE_ADDR': _WHITELISTED_IP}) job = job_key.get() self.assertEqual(6790, job.results_data['try_job_id']) self.assertEqual('started', job.results_data['status'])
def testCreateTryJob_WithoutExistingBug(self): # Put succeeded job in the datastore. try_job.TryJob( bug_id=12345, status='started', bot='win_perf', results_data=_SAMPLE_BISECT_RESULTS_JSON).put() self.testapp.get('/update_bug_with_results') pending_jobs = try_job.TryJob.query().fetch() # Expects job to finish. self.assertEqual(1, len(pending_jobs)) self.assertEqual(12345, pending_jobs[0].bug_id) self.assertEqual('completed', pending_jobs[0].status)
def testCreateTryJob_WithoutExistingBug(self): # Put succeeded job in the datastore. try_job.TryJob(bug_id=12345, rietveld_issue_id=200034, rietveld_patchset_id=1, status='started', bot='win_perf').put() self.testapp.get('/update_bug_with_results') pending_jobs = try_job.TryJob.query().fetch() # Expects job to finish. self.assertEqual(0, len(pending_jobs))
def testValidateBuildbucketResponse_Success(self): buildbucket_response_success = r"""{ "build": { "status": "COMPLETED", "url": "http://build.chromium.org/linux_perf_bisector/builds/47", "id": "9043278384371361584", "result": "SUCCESS" } }""" job = try_job.TryJob(bug_id=12345, status='started', bot='win_perf') job.put() self.assertTrue(update_bug_with_results._ValidateBuildbucketResponse( json.loads(buildbucket_response_success)))
def testPost(self): try_job.TryJob(bug_id=12345, status='started', bot='win_perf', results_data={}).put() try_job.TryJob(bug_id=12345, status='failed', bot='android_bisect', results_data={ 'metric': 'foo' }).put() try_job.TryJob(bug_id=99999, status='failed', bot='win_perf', results_data={ 'metric': 'foo' }).put() response = self.testapp.post('/bug_details', {'bug_id': '12345'}) self.assertEqual('Regression in sunspider', self.GetJsonValue(response, 'summary')) self.assertEqual('*****@*****.**', self.GetJsonValue(response, 'owner')) self.assertEqual('2017-02-17T23:08:44', self.GetJsonValue(response, 'published')) self.assertEqual('open', self.GetJsonValue(response, 'state')) self.assertEqual('Untriaged', self.GetJsonValue(response, 'status')) comments = self.GetJsonValue(response, 'comments') self.assertEqual(3, len(comments)) self.assertEqual('This is the second comment', comments[1]['content']) self.assertItemsEqual(['https://codereview.chromium.org/2707483002'], self.GetJsonValue(response, 'review_urls')) bisects = self.GetJsonValue(response, 'bisects') self.assertEqual(2, len(bisects)) self.assertEqual('started', bisects[0]['status']) self.assertIsNone(bisects[0]['metric']) self.assertEqual('failed', bisects[1]['status']) self.assertEqual('foo', bisects[1]['metric']) self.assertEqual('android_bisect', bisects[1]['bot'])
def _PerformBisectStep(self, user): """Gathers the parameters for a bisect job and triggers the job.""" bug_id = int(self.request.get('bug_id', -1)) master_name = self.request.get('master', 'ChromiumPerf') internal_only = self.request.get('internal_only') == 'true' bisect_bot = self.request.get('bisect_bot') bypass_no_repro_check = self.request.get( 'bypass_no_repro_check') == 'true' use_staging_bot = self.request.get('use_staging_bot') == 'true' if use_staging_bot: bisect_bot = _GuessStagingBot(master_name, bisect_bot) or bisect_bot bisect_config = GetBisectConfig( bisect_bot=bisect_bot, master_name=master_name, suite=self.request.get('suite'), metric=self.request.get('metric'), good_revision=self.request.get('good_revision'), bad_revision=self.request.get('bad_revision'), repeat_count=self.request.get('repeat_count', 10), max_time_minutes=self.request.get('max_time_minutes', 20), bug_id=bug_id, story_filter=self.request.get('story_filter'), use_archive=self.request.get('use_archive'), bisect_mode=self.request.get('bisect_mode', 'mean'), bypass_no_repro_check=bypass_no_repro_check) if 'error' in bisect_config: return bisect_config config_python_string = 'config = %s\n' % json.dumps( bisect_config, sort_keys=True, indent=2, separators=(',', ': ')) bisect_job = try_job.TryJob(bot=bisect_bot, config=config_python_string, bug_id=bug_id, email=user.email(), master_name=master_name, internal_only=internal_only, job_type='bisect') try: results = PerformBisect(bisect_job) except request_handler.InvalidInputError as iie: results = {'error': iie.message} if 'error' in results and bisect_job.key: bisect_job.key.delete() return results
def _PerformBisectStep(self, user): """Gathers the parameters for a bisect job and triggers the job.""" bug_id = int(self.request.get('bug_id', -1)) master_name = self.request.get('master', 'ChromiumPerf') internal_only = self.request.get('internal_only') == 'true' bisect_bot = self.request.get('bisect_bot') use_staging_bot = self.request.get('use_staging_bot') == 'true' bisect_config = GetBisectConfig( bisect_bot=bisect_bot, master_name=master_name, suite=self.request.get('suite'), metric=self.request.get('metric'), good_revision=self.request.get('good_revision'), bad_revision=self.request.get('bad_revision'), repeat_count=self.request.get('repeat_count', 10), max_time_minutes=self.request.get('max_time_minutes', 20), bug_id=bug_id, story_filter=self.request.get('story_filter'), bisect_mode=self.request.get('bisect_mode', 'mean'), use_staging_bot=use_staging_bot) if 'error' in bisect_config: return bisect_config config_python_string = 'config = %s\n' % json.dumps( bisect_config, sort_keys=True, indent=2, separators=(',', ': ')) bisect_job = try_job.TryJob( bot=bisect_bot, config=config_python_string, bug_id=bug_id, email=user, master_name=master_name, internal_only=internal_only, job_type='bisect') alert_keys = self.request.get('alerts') alerts = None if alert_keys: alerts = [ndb.Key(urlsafe=a).get() for a in json.loads(alert_keys)] try: results = PerformBisect(bisect_job, alerts=alerts) except request_handler.InvalidInputError as iie: results = {'error': iie.message} if 'error' in results and bisect_job.key: bisect_job.key.delete() return results
def testPost_FailedJobRunTwice_JobRestarted(self, mock_perform_bisect): testing_common.AddTests( ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}}) test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score') anomaly.Anomaly( bug_id=111, test=test_key, start_revision=300100, end_revision=300200, median_before_anomaly=100, median_after_anomaly=200).put() try_job.TryJob( bug_id=111, status='failed', last_ran_timestamp=datetime.datetime.now() - datetime.timedelta(days=8), run_count=2).put() self.testapp.post('/auto_bisect') mock_perform_bisect.assert_called_once_with( try_job.TryJob.query(try_job.TryJob.bug_id == 111).get())