def testStartNewBisectForBug_Pinpoint_ParamsRaisesError(self): testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'], {'sunspider': { 'score': {} }}) test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score') testing_common.AddRows( 'ChromiumPerf/linux-pinpoint/sunspider/score', { 11999: { 'r_foo': '9e29b5bcd08357155b2859f87227d50ed60cf857' }, 12500: { 'r_foo': 'fc34e5346446854637311ad7793a95d56e314042' } }) anomaly.Anomaly(bug_id=333, test=test_key, start_revision=12000, end_revision=12501, median_before_anomaly=100, median_after_anomaly=200).put() result = auto_bisect.StartNewBisectForBug(333) self.assertEqual({'error': 'Some reason'}, result)
def testPinpointParams_GitHashes(self, mock_commit): def _MockCommit(git_sha): if git_sha == 'abc': return {'number': 1050} else: return {'number': 1150} mock_commit.side_effect = _MockCommit testing_common.AddTests(['ChromiumPerf'], ['Android Nexus5X WebView Perf'], {'system_health': { 'foo': {} }}) params = { 'test_path': 'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo', 'start_commit': 'abc', 'end_commit': 'def', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } t = graph_data.TestMetadata(id=params['test_path']) t.put() rows = dict( itertools.chain( zip(itertools.islice(itertools.count(1000, 2), 50), itertools.repeat({'value': 0.1})), zip(itertools.islice(itertools.count(1101, 2), 50), itertools.repeat({'value': 0.5})))) testing_common.AddRows(params['test_path'], rows) results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual(0.4, results['comparison_magnitude'])
def testMakeAnomalyEntity_RevisionRanges(self): testing_common.AddTests( ['ClankInternal'], ['linux'], { 'page_cycler_v2': { 'cnn': {}, 'cnn_ref': {}, 'yahoo': {}, 'nytimes': {}, }, }) test = utils.TestKey('ClankInternal/linux/page_cycler_v2/cnn').get() testing_common.AddRows(test.test_path, [100, 200, 300, 400]) for row in graph_data.Row.query(): row.r_commit_pos = int(row.value) + 2 # Different enough to ensure it is # picked up properly. row.put() alert = find_anomalies._MakeAnomalyEntity( _MakeSampleChangePoint(300, 50, 100), test, list(graph_data.Row.query())) self.assertEqual(alert.display_start, 203) self.assertEqual(alert.display_end, 302)
def testStartNewBisectForBug_WithDefaultRevs_StartsBisect( self, mock_perform_bisect): testing_common.AddTests( ['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}}) test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score') testing_common.AddRows( 'ChromiumPerf/linux-release/sunspider/score', { 11990: { 'a_default_rev': 'r_foo', 'r_foo': '9e29b5bcd08357155b2859f87227d50ed60cf857' }, 12500: { 'a_default_rev': 'r_foo', 'r_foo': 'fc34e5346446854637311ad7793a95d56e314042' } }) anomaly.Anomaly( bug_id=333, test=test_key, start_revision=12000, end_revision=12500, median_before_anomaly=100, median_after_anomaly=200).put() auto_bisect.StartNewBisectForBug(333) job = try_job.TryJob.query(try_job.TryJob.bug_id == 333).get() mock_perform_bisect.assert_called_once_with(job)
def _AddData(self): """Adds sample TestMetadata entities and returns their keys.""" testing_common.AddTests(['ChromiumPerf'], ['linux'], {'page_cycler': { 'warm': { 'cnn': {}, } }}) test_path = 'ChromiumPerf/linux/page_cycler/warm/cnn' test = utils.TestKey(test_path).get() test.improvement_direction = anomaly.UP test.put() now = datetime.datetime.now() last_week = now - datetime.timedelta(days=7) rows = dict([(i * 100, { 'value': i * 1000, 'a_whatever': 'blah', 'r_v8': '1234a', 'timestamp': now if i > 5 else last_week, 'error': 3.3232 }) for i in range(1, 10)]) rows[100]['r_not_every_row'] = 12345 testing_common.AddRows('ChromiumPerf/linux/page_cycler/warm/cnn', rows)
def testPinpointParams_NoAlert(self): params = { 'test_path': 'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo', 'start_commit': '1051', 'end_commit': '1151', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } t = graph_data.TestMetadata(id=params['test_path']) t.put() rows = {} for i in xrange(1000, 1100, 2): rows[i] = {'value': 0.1} for i in xrange(1101, 1200, 2): rows[i] = {'value': 0.5} testing_common.AddRows(params['test_path'], rows) results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual(0.4, results['comparison_magnitude'])
def _AddTestData(self, series, sheriff_key, improvement_direction=anomaly.UP): """Adds one sample TestMetadata and associated data. Args: series: Either a list of values, or a list of (x, y) pairs. sheriff_key: A Sheriff entity key. improvement_direction: One of {anomaly.UP, anomaly.DOWN, anomaly.UNKNOWN}. Returns: The key of the TestMetadata entity that was added. """ testing_common.AddTests(['M'], ['b'], {'benchmark': {'t': {}}}) test_path = 'M/b/benchmark/t' test = utils.TestKey(test_path).get() test.improvement_direction = improvement_direction test.sheriff = sheriff_key sheriff_entity = sheriff_key.get() sheriff_entity.patterns.append(test.test_path) sheriff_entity.put() if series and isinstance(series[0], (int, float)): series = enumerate(series, start=1) testing_common.AddRows(test_path, {x: {'value': y} for x, y in series}) return test.put()
def testPinpointParams_OverriddenAnomalyConfig(self): params = { 'test_path': 'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo', 'start_commit': '1051', 'end_commit': '1151', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } a = anomaly_config.AnomalyConfig() a.config = {'min_segment_size': 1} a.patterns = ['*/*/*/*'] a.put() t = graph_data.TestMetadata(id=params['test_path']) t.overridden_anomaly_config = a.key t.put() rows = dict( itertools.chain( zip( itertools.islice(itertools.count(1000, 2), 75), itertools.repeat({'value': -100.0})), [(1050, {'value': 0.1})], zip( itertools.islice(itertools.count(1101, 2), 50), itertools.repeat({'value': 0.5})))) testing_common.AddRows(params['test_path'], rows) results = pinpoint_request.PinpointParamsFromBisectParams(params) # We overrode the anomaly config with a window of 1, and there's only a # single row with value 0.1, the rest are 0.0. self.assertEqual(0.4, results['comparison_magnitude'])
def _AddRows(self, keys): for key in keys: testing_common.AddRows(utils.TestPath(key), [1, 2, 3, RECENT_REV])
def testStartNewBisectForBut_BlacklistedMaster_SucceedsIfAlternative( self, _, mock_new): # Even if there are blacklisted masters, the bisect request should still # succeed if there's a non-blacklisted master. namespaced_stored_object.Set('file_bug_bisect_blacklist', {'ChromiumPerf': []}) namespaced_stored_object.Set('bot_configurations', { 'linux-pinpoint': { 'dimensions': [{ 'key': 'foo', 'value': 'bar' }] }, }) namespaced_stored_object.Set('repositories', { 'chromium': { 'some': 'params' }, }) mock_new.return_value = {'jobId': 123, 'jobUrl': 'http://pinpoint/123'} testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'], {'sunspider': { 'score': {} }}) testing_common.AddTests(['ChromiumPerf2'], ['linux-pinpoint'], {'sunspider': { 'score': {} }}) test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score') test_key2 = utils.TestKey( 'ChromiumPerf2/linux-pinpoint/sunspider/score') rows = { 11999: { 'a_default_rev': 'r_chromium', 'r_chromium': '9e29b5bcd08357155b2859f87227d50ed60cf857' }, 12500: { 'a_default_rev': 'r_chromium', 'r_chromium': 'fc34e5346446854637311ad7793a95d56e314042' } } testing_common.AddRows('ChromiumPerf/linux-pinpoint/sunspider/score', rows) testing_common.AddRows('ChromiumPerf2/linux-pinpoint/sunspider/score', rows) anomaly.Anomaly(bug_id=333, test=test_key, start_revision=12000, end_revision=12500, median_before_anomaly=100, median_after_anomaly=200).put() a = anomaly.Anomaly(bug_id=333, test=test_key2, start_revision=12000, end_revision=12500, median_before_anomaly=100, median_after_anomaly=200).put() result = auto_bisect.StartNewBisectForBug(333) self.assertNotIn('error', result) self.assertEqual({ 'alert': a.urlsafe(), 'test_path': test_key2.id() }, json.loads(mock_new.call_args[0][0]['tags']))
def testMakeAnomalyEntity_AddsOwnership(self): data_samples = [{ 'type': 'GenericSet', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'values': ['*****@*****.**', '*****@*****.**'] }, { 'type': 'GenericSet', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', 'values': ['abc'] }, { 'type': 'GenericSet', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb828', 'values': ['This is an info blurb.'] }] test_key = utils.TestKey('ChromiumPerf/linux/page_cycler_v2/cnn') testing_common.AddTests( ['ChromiumPerf'], ['linux'], { 'page_cycler_v2': { 'cnn': {}, 'cnn_ref': {}, 'yahoo': {}, 'nytimes': {}, }, }) test = test_key.get() testing_common.AddRows(test.test_path, [100, 200, 300, 400]) suite_key = utils.TestKey('ChromiumPerf/linux/page_cycler_v2') entity = histogram.SparseDiagnostic(data=data_samples[0], test=suite_key, start_revision=1, end_revision=sys.maxsize, id=data_samples[0]['guid'], name=reserved_infos.OWNERS.name) entity.put() entity = histogram.SparseDiagnostic( data=data_samples[1], test=suite_key, start_revision=1, end_revision=sys.maxsize, id=data_samples[1]['guid'], name=reserved_infos.BUG_COMPONENTS.name) entity.put() entity = histogram.SparseDiagnostic( data=data_samples[2], test=suite_key, start_revision=1, end_revision=sys.maxsize, id=data_samples[2]['guid'], name=reserved_infos.INFO_BLURB.name) entity.put() alert = find_anomalies._MakeAnomalyEntity( _MakeSampleChangePoint(10011, 50, 100), test, 'avg', self._DataSeries()).get_result() self.assertEqual(alert.ownership['component'], 'abc') self.assertListEqual(alert.ownership['emails'], ['*****@*****.**', '*****@*****.**']) self.assertEqual(alert.ownership['info_blurb'], 'This is an info blurb.')
def _AddRows(self, keys): for key in keys: testing_common.AddRows(utils.TestPath(key), [1, 2, 3, 445588])
def testStartNewBisectForBug_Pinpoint_Succeeds(self, mock_new): namespaced_stored_object.Set('bot_configurations', { 'linux-pinpoint': { 'dimensions': [{ 'key': 'foo', 'value': 'bar' }] }, }) namespaced_stored_object.Set('repositories', { 'chromium': { 'some': 'params' }, }) mock_new.return_value = {'jobId': 123, 'jobUrl': 'http://pinpoint/123'} testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'], {'sunspider': { 'score': {} }}) test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score') testing_common.AddRows( 'ChromiumPerf/linux-pinpoint/sunspider/score', { 11999: { 'a_default_rev': 'r_chromium', 'r_chromium': '9e29b5bcd08357155b2859f87227d50ed60cf857' }, 12500: { 'a_default_rev': 'r_chromium', 'r_chromium': 'fc34e5346446854637311ad7793a95d56e314042' } }) t = test_key.get() t.unescaped_story_name = 'score:foo' t.put() a = anomaly.Anomaly(bug_id=333, project_id='test_project', test=test_key, start_revision=12000, end_revision=12500, median_before_anomaly=100, median_after_anomaly=200).put() result = auto_bisect.StartNewBisectForBug(333, 'test_project') self.assertEqual({ 'issue_id': 123, 'issue_url': 'http://pinpoint/123' }, result) self.assertEqual('123', a.get().pinpoint_bisects[0]) self.assertEqual({ 'alert': a.urlsafe(), 'test_path': test_key.id() }, json.loads(mock_new.call_args[0][0]['tags'])) self.assertEqual('score:foo', mock_new.call_args[0][0]['story']) anomaly_entity = a.get() anomaly_magnitude = (anomaly_entity.median_after_anomaly - anomaly_entity.median_before_anomaly) self.assertEqual(anomaly_magnitude, mock_new.call_args[0][0]['comparison_magnitude'])