def testPinpointParams_Metric_ChartAndTrace(self): params = { 'test_path': 'ChromiumPerf/mac/blink_perf/foo/http___bar.html', 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'start_repository': 'chromium', 'end_repository': 'chromium', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', } graph_data.TestMetadata( id=params['test_path'], unescaped_story_name='http://bar.html').put() results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('', results['tir_label']) self.assertEqual('foo', results['chart']) self.assertEqual('http://bar.html', results['trace'])
def testPinpointParams_Metric_TIRLabelChartAndTrace(self): params = { 'test_path': 'ChromiumPerf/mac/blink_perf/foo/label/bar.html', 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } t = graph_data.TestMetadata(id=params['test_path'], ) t.UpdateSheriff() t.put() results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('label', results['tir_label']) self.assertEqual('foo', results['chart']) self.assertEqual('bar.html', results['trace'])
def testPinpointParams_SplitsStatistics(self): statistic_types = ['avg', 'min', 'max', 'sum', 'std', 'count'] for s in statistic_types: testing_common.AddTests( ['ChromiumPerf'], ['mac'], {'system_health': {'foo_%s' % s: {}}}) params = { 'test_path': 'ChromiumPerf/mac/system_health/foo_%s' % s, 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', 'bug_id': -1, } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual(s, results['statistic']) self.assertEqual('foo', results['chart'])
def testPinpointParams_ConvertsCommitsToGitHashes(self): testing_common.AddTests(['ChromiumPerf'], ['android-webview-nexus5x'], {'system_health': { 'foo': {} }}) params = { 'test_path': 'ChromiumPerf/android-webview-nexus5x/system_health/foo', 'start_commit': '1234', 'end_commit': '5678', 'bug_id': '', 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('abcd', results['start_git_hash']) self.assertEqual('abcd', results['end_git_hash'])
def testPinpointParams_NoData(self): testing_common.AddTests( ['ChromiumPerf'], ['Android Nexus5X WebView Perf'], {'system_health': {'foo': {}}}) params = { 'test_path': 'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo', 'start_commit': '1050', 'end_commit': '1150', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } t = graph_data.TestMetadata(id=params['test_path']) t.put() results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertFalse('comparison_magnitude' in results)
def testPinpointParams_V8(self, mock_crrev): mock_crrev.return_value = {'git_sha': 'acbd'} params = { 'test_path': 'internal.client.v8/Pixel2/v8/JSTests/Array/Total', 'start_commit': '1234', 'end_commit': '5678', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': 'https://path/to/patch', } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertNotIn('tir_label', results) self.assertNotIn('trace', results) self.assertEqual('', results['chart']) self.assertEqual('', results['target']) mock_crrev.assert_any_call( number='1234', numbering_identifier='refs/heads/master', numbering_type='COMMIT_POSITION', project='chromium', repo='v8/v8')
def testPinpointParams_IsolateTarget_WebviewTelemetry(self): params = { 'test_path': 'ChromiumPerf/android-webview-nexus5x/system_health/foo', 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('android-webview-nexus5x', results['configuration']) self.assertEqual('system_health', results['benchmark']) self.assertEqual('foo', results['chart']) self.assertEqual('telemetry_perf_webview_tests', results['target']) self.assertEqual('*****@*****.**', results['user']) self.assertEqual('abcd1234', results['start_git_hash']) self.assertEqual('efgh5678', results['end_git_hash']) self.assertEqual('true', results['auto_explore']) self.assertEqual(1, results['bug_id'])
def testPinpointParams_Metric_ChartAndTrace(self): testing_common.AddTests( ['ChromiumPerf'], ['mac'], {'blink_perf': {'foo': {'http___bar.html': {}}}}) params = { 'test_path': 'ChromiumPerf/mac/blink_perf/foo/http___bar.html', 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } t = graph_data.TestMetadata( id=params['test_path'], unescaped_story_name='http://bar.html') t.UpdateSheriff() t.put() results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('foo', results['chart']) self.assertEqual('http://bar.html', results['trace'])
def testPinpointParams_OverriddenAnomalyConfig(self): testing_common.AddTests(['ChromiumPerf'], ['Android Nexus5X WebView Perf'], {'system_health': { 'foo': {} }}) params = { 'test_path': 'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo', 'start_commit': '1051', 'end_commit': '1151', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } a = anomaly_config.AnomalyConfig() a.config = {'min_segment_size': 1} a.patterns = ['*/*/*/*'] a.put() t = graph_data.TestMetadata(id=params['test_path']) t.overridden_anomaly_config = a.key t.put() rows = dict( itertools.chain( zip(itertools.islice(itertools.count(1000, 2), 75), itertools.repeat({'value': -100.0})), [(1050, { 'value': 0.1 })], zip(itertools.islice(itertools.count(1101, 2), 50), itertools.repeat({'value': 0.5})))) testing_common.AddRows(params['test_path'], rows) results = pinpoint_request.PinpointParamsFromBisectParams(params) # We overrode the anomaly config with a window of 1, and there's only a # single row with value 0.1, the rest are 0.0. self.assertEqual(0.4, results['comparison_magnitude'])
def testPinpointParams_ComparisonMagnitude_Added(self): test_key = utils.TestKey('ChromiumPerf/mac/cc_perftests/foo') anomaly_entity = anomaly.Anomaly( start_revision=1, end_revision=2, test=test_key, median_before_anomaly=1, median_after_anomaly=10) anomaly_entity.put() params = { 'test_path': 'ChromiumPerf/mac/cc_perftests/foo', 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', 'alerts': json.dumps([anomaly_entity.key.urlsafe()]) } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual(9, results['comparison_magnitude']) self.assertEqual( anomaly_entity.key.urlsafe(), json.loads(results['tags'])['alert'])
def testPinpointParams_IsolateTarget_Telemetry(self): params = { 'test_path': 'ChromiumPerf/mac/system_health/foo', 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'bug_id': 1, 'story_filter': 'foo', 'pin': '', 'bisect_mode': 'performance', } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('mac', results['configuration']) self.assertEqual('system_health', results['benchmark']) self.assertEqual('foo', results['chart']) self.assertEqual('performance_test_suite', results['target']) self.assertEqual('*****@*****.**', results['user']) self.assertEqual('abcd1234', results['start_git_hash']) self.assertEqual('efgh5678', results['end_git_hash']) self.assertEqual('performance', results['comparison_mode']) self.assertEqual(1, results['bug_id']) self.assertEqual('foo', results['story'])
def testPinpointParams_IsolateTarget_NonTelemetry(self): params = { 'test_path': 'ChromiumPerf/mac/cc_perftests/foo', 'start_git_hash': 'abcd1234', 'end_git_hash': 'efgh5678', 'start_repository': 'chromium', 'end_repository': 'chromium', 'bug_id': 1, } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('cc_perftests', results['target']) self.assertEqual('*****@*****.**', results['email']) self.assertEqual('chromium', results['start_repository']) self.assertEqual('abcd1234', results['start_git_hash']) self.assertEqual('chromium', results['end_repository']) self.assertEqual('efgh5678', results['end_git_hash']) self.assertEqual('1', results['auto_explore']) self.assertEqual(1, results['bug_id']) self.assertEqual([{ 'key': 'foo', 'value': 'mac_dimensions' }], json.loads(results['dimensions']))
def testPinpointParams_GitHashes(self, mock_commit): def _MockCommit(git_sha): if git_sha == 'abc': return {'number': 1050} else: return {'number': 1150} mock_commit.side_effect = _MockCommit testing_common.AddTests(['ChromiumPerf'], ['Android Nexus5X WebView Perf'], {'system_health': { 'foo': {} }}) params = { 'test_path': 'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo', 'start_commit': 'abc', 'end_commit': 'def', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } t = graph_data.TestMetadata(id=params['test_path']) t.put() rows = dict( itertools.chain( zip(itertools.islice(itertools.count(1000, 2), 50), itertools.repeat({'value': 0.1})), zip(itertools.islice(itertools.count(1101, 2), 50), itertools.repeat({'value': 0.5})))) testing_common.AddRows(params['test_path'], rows) results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual(0.4, results['comparison_magnitude'])
def testPinpointParams_NoAlert(self): params = { 'test_path': 'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo', 'start_commit': '1051', 'end_commit': '1151', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'pin': '', } t = graph_data.TestMetadata(id=params['test_path']) t.put() rows = {} for i in xrange(1000, 1100, 2): rows[i] = {'value': 0.1} for i in xrange(1101, 1200, 2): rows[i] = {'value': 0.5} testing_common.AddRows(params['test_path'], rows) results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual(0.4, results['comparison_magnitude'])
def testPinpointParams_IsolateTarget_WebviewTelemetry(self): params = { 'test_path': 'ChromiumPerf/android-webview-nexus5x/system_health/foo', 'start_git_hash': 'abcd1234', 'end_git_hash': 'efgh5678', 'start_repository': 'chromium', 'end_repository': 'chromium', 'bug_id': 1 } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('telemetry_perf_webview_tests', results['target']) self.assertEqual('*****@*****.**', results['email']) self.assertEqual('chromium', results['start_repository']) self.assertEqual('abcd1234', results['start_git_hash']) self.assertEqual('chromium', results['end_repository']) self.assertEqual('efgh5678', results['end_git_hash']) self.assertEqual('1', results['auto_explore']) self.assertEqual(1, results['bug_id']) self.assertEqual([{ 'key': 'foo', 'value': 'android_dimensions' }], json.loads(results['dimensions']))
def testPinpointParams_IsolateTarget_NonTelemetry(self): params = { 'test_path': 'ChromiumPerf/mac/cc_perftests/foo', 'start_commit': 'abcd1234', 'end_commit': 'efgh5678', 'bug_id': 1, 'bisect_mode': 'performance', 'story_filter': '', 'alerts': json.dumps(['123']) } results = pinpoint_request.PinpointParamsFromBisectParams(params) self.assertEqual('mac', results['configuration']) self.assertEqual('cc_perftests', results['benchmark']) self.assertEqual('foo', results['chart']) self.assertEqual('cc_perftests', results['target']) self.assertEqual('*****@*****.**', results['user']) self.assertEqual('abcd1234', results['start_git_hash']) self.assertEqual('efgh5678', results['end_git_hash']) self.assertEqual('true', results['auto_explore']) self.assertEqual(1, results['bug_id']) self.assertEqual(params['test_path'], json.loads(results['tags'])['test_path']) self.assertEqual('123', json.loads(results['tags'])['alert'])
def testPinpointParams_InvalidSheriff_RaisesError(self): params = {'test_path': 'ChromiumPerf/foo/blah/foo'} with self.assertRaises(pinpoint_request.InvalidParamsError): pinpoint_request.PinpointParamsFromBisectParams(params)
def testPinpointParams_BotUndefined_ReturnsError(self): params = {'test_path': 'ChromiumPerf/foo/blah/foo'} with self.assertRaises(pinpoint_request.InvalidParamsError): pinpoint_request.PinpointParamsFromBisectParams(params)