def testPinpointParams_Metric_ChartAndTrace(self):
params = {
'test_path': 'ChromiumPerf/mac/blink_perf/foo/http___bar.html',
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'start_repository': 'chromium',
'end_repository': 'chromium',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
}
graph_data.TestMetadata(
id=params['test_path'], unescaped_story_name='http://bar.html').put()
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('', results['tir_label'])
self.assertEqual('foo', results['chart'])
self.assertEqual('http://bar.html', results['trace'])
def testPinpointParams_Metric_TIRLabelChartAndTrace(self):
params = {
'test_path': 'ChromiumPerf/mac/blink_perf/foo/label/bar.html',
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
t = graph_data.TestMetadata(id=params['test_path'], )
t.UpdateSheriff()
t.put()
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('label', results['tir_label'])
self.assertEqual('foo', results['chart'])
self.assertEqual('bar.html', results['trace'])
def testPinpointParams_SplitsStatistics(self):
statistic_types = ['avg', 'min', 'max', 'sum', 'std', 'count']
for s in statistic_types:
testing_common.AddTests(
['ChromiumPerf'], ['mac'], {'system_health': {'foo_%s' % s: {}}})
params = {
'test_path': 'ChromiumPerf/mac/system_health/foo_%s' % s,
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
'bug_id': -1,
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual(s, results['statistic'])
self.assertEqual('foo', results['chart'])
def testPinpointParams_ConvertsCommitsToGitHashes(self):
testing_common.AddTests(['ChromiumPerf'], ['android-webview-nexus5x'],
{'system_health': {
'foo': {}
}})
params = {
'test_path':
'ChromiumPerf/android-webview-nexus5x/system_health/foo',
'start_commit': '1234',
'end_commit': '5678',
'bug_id': '',
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('abcd', results['start_git_hash'])
self.assertEqual('abcd', results['end_git_hash'])
def testPinpointParams_NoData(self):
testing_common.AddTests(
['ChromiumPerf'], ['Android Nexus5X WebView Perf'],
{'system_health': {'foo': {}}})
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': '1050',
'end_commit': '1150',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
t = graph_data.TestMetadata(id=params['test_path'])
t.put()
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertFalse('comparison_magnitude' in results)
def testPinpointParams_V8(self, mock_crrev):
mock_crrev.return_value = {'git_sha': 'acbd'}
params = {
'test_path': 'internal.client.v8/Pixel2/v8/JSTests/Array/Total',
'start_commit': '1234',
'end_commit': '5678',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': 'https://path/to/patch',
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertNotIn('tir_label', results)
self.assertNotIn('trace', results)
self.assertEqual('', results['chart'])
self.assertEqual('', results['target'])
mock_crrev.assert_any_call(
number='1234', numbering_identifier='refs/heads/master',
numbering_type='COMMIT_POSITION', project='chromium', repo='v8/v8')
def testPinpointParams_IsolateTarget_WebviewTelemetry(self):
params = {
'test_path':
'ChromiumPerf/android-webview-nexus5x/system_health/foo',
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('android-webview-nexus5x', results['configuration'])
self.assertEqual('system_health', results['benchmark'])
self.assertEqual('foo', results['chart'])
self.assertEqual('telemetry_perf_webview_tests', results['target'])
self.assertEqual('*****@*****.**', results['user'])
self.assertEqual('abcd1234', results['start_git_hash'])
self.assertEqual('efgh5678', results['end_git_hash'])
self.assertEqual('true', results['auto_explore'])
self.assertEqual(1, results['bug_id'])
def testPinpointParams_Metric_ChartAndTrace(self):
testing_common.AddTests(
['ChromiumPerf'], ['mac'],
{'blink_perf': {'foo': {'http___bar.html': {}}}})
params = {
'test_path': 'ChromiumPerf/mac/blink_perf/foo/http___bar.html',
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
t = graph_data.TestMetadata(
id=params['test_path'], unescaped_story_name='http://bar.html')
t.UpdateSheriff()
t.put()
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('foo', results['chart'])
self.assertEqual('http://bar.html', results['trace'])
def testPinpointParams_OverriddenAnomalyConfig(self):
testing_common.AddTests(['ChromiumPerf'],
['Android Nexus5X WebView Perf'],
{'system_health': {
'foo': {}
}})
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': '1051',
'end_commit': '1151',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
a = anomaly_config.AnomalyConfig()
a.config = {'min_segment_size': 1}
a.patterns = ['*/*/*/*']
a.put()
t = graph_data.TestMetadata(id=params['test_path'])
t.overridden_anomaly_config = a.key
t.put()
rows = dict(
itertools.chain(
zip(itertools.islice(itertools.count(1000, 2), 75),
itertools.repeat({'value': -100.0})), [(1050, {
'value': 0.1
})],
zip(itertools.islice(itertools.count(1101, 2), 50),
itertools.repeat({'value': 0.5}))))
testing_common.AddRows(params['test_path'], rows)
results = pinpoint_request.PinpointParamsFromBisectParams(params)
# We overrode the anomaly config with a window of 1, and there's only a
# single row with value 0.1, the rest are 0.0.
self.assertEqual(0.4, results['comparison_magnitude'])
def testPinpointParams_ComparisonMagnitude_Added(self):
test_key = utils.TestKey('ChromiumPerf/mac/cc_perftests/foo')
anomaly_entity = anomaly.Anomaly(
start_revision=1, end_revision=2, test=test_key,
median_before_anomaly=1, median_after_anomaly=10)
anomaly_entity.put()
params = {
'test_path': 'ChromiumPerf/mac/cc_perftests/foo',
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
'alerts': json.dumps([anomaly_entity.key.urlsafe()])
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual(9, results['comparison_magnitude'])
self.assertEqual(
anomaly_entity.key.urlsafe(), json.loads(results['tags'])['alert'])
def testPinpointParams_IsolateTarget_Telemetry(self):
params = {
'test_path': 'ChromiumPerf/mac/system_health/foo',
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'bug_id': 1,
'story_filter': 'foo',
'pin': '',
'bisect_mode': 'performance',
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('mac', results['configuration'])
self.assertEqual('system_health', results['benchmark'])
self.assertEqual('foo', results['chart'])
self.assertEqual('performance_test_suite', results['target'])
self.assertEqual('*****@*****.**', results['user'])
self.assertEqual('abcd1234', results['start_git_hash'])
self.assertEqual('efgh5678', results['end_git_hash'])
self.assertEqual('performance', results['comparison_mode'])
self.assertEqual(1, results['bug_id'])
self.assertEqual('foo', results['story'])
def testPinpointParams_IsolateTarget_NonTelemetry(self):
params = {
'test_path': 'ChromiumPerf/mac/cc_perftests/foo',
'start_git_hash': 'abcd1234',
'end_git_hash': 'efgh5678',
'start_repository': 'chromium',
'end_repository': 'chromium',
'bug_id': 1,
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('cc_perftests', results['target'])
self.assertEqual('*****@*****.**', results['email'])
self.assertEqual('chromium', results['start_repository'])
self.assertEqual('abcd1234', results['start_git_hash'])
self.assertEqual('chromium', results['end_repository'])
self.assertEqual('efgh5678', results['end_git_hash'])
self.assertEqual('1', results['auto_explore'])
self.assertEqual(1, results['bug_id'])
self.assertEqual([{
'key': 'foo',
'value': 'mac_dimensions'
}], json.loads(results['dimensions']))
def testPinpointParams_GitHashes(self, mock_commit):
def _MockCommit(git_sha):
if git_sha == 'abc':
return {'number': 1050}
else:
return {'number': 1150}
mock_commit.side_effect = _MockCommit
testing_common.AddTests(['ChromiumPerf'],
['Android Nexus5X WebView Perf'],
{'system_health': {
'foo': {}
}})
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': 'abc',
'end_commit': 'def',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
t = graph_data.TestMetadata(id=params['test_path'])
t.put()
rows = dict(
itertools.chain(
zip(itertools.islice(itertools.count(1000, 2), 50),
itertools.repeat({'value': 0.1})),
zip(itertools.islice(itertools.count(1101, 2), 50),
itertools.repeat({'value': 0.5}))))
testing_common.AddRows(params['test_path'], rows)
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual(0.4, results['comparison_magnitude'])
def testPinpointParams_NoAlert(self):
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': '1051',
'end_commit': '1151',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
t = graph_data.TestMetadata(id=params['test_path'])
t.put()
rows = {}
for i in xrange(1000, 1100, 2):
rows[i] = {'value': 0.1}
for i in xrange(1101, 1200, 2):
rows[i] = {'value': 0.5}
testing_common.AddRows(params['test_path'], rows)
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual(0.4, results['comparison_magnitude'])
def testPinpointParams_IsolateTarget_WebviewTelemetry(self):
params = {
'test_path':
'ChromiumPerf/android-webview-nexus5x/system_health/foo',
'start_git_hash': 'abcd1234',
'end_git_hash': 'efgh5678',
'start_repository': 'chromium',
'end_repository': 'chromium',
'bug_id': 1
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('telemetry_perf_webview_tests', results['target'])
self.assertEqual('*****@*****.**', results['email'])
self.assertEqual('chromium', results['start_repository'])
self.assertEqual('abcd1234', results['start_git_hash'])
self.assertEqual('chromium', results['end_repository'])
self.assertEqual('efgh5678', results['end_git_hash'])
self.assertEqual('1', results['auto_explore'])
self.assertEqual(1, results['bug_id'])
self.assertEqual([{
'key': 'foo',
'value': 'android_dimensions'
}], json.loads(results['dimensions']))
def testPinpointParams_IsolateTarget_NonTelemetry(self):
params = {
'test_path': 'ChromiumPerf/mac/cc_perftests/foo',
'start_commit': 'abcd1234',
'end_commit': 'efgh5678',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'alerts': json.dumps(['123'])
}
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual('mac', results['configuration'])
self.assertEqual('cc_perftests', results['benchmark'])
self.assertEqual('foo', results['chart'])
self.assertEqual('cc_perftests', results['target'])
self.assertEqual('*****@*****.**', results['user'])
self.assertEqual('abcd1234', results['start_git_hash'])
self.assertEqual('efgh5678', results['end_git_hash'])
self.assertEqual('true', results['auto_explore'])
self.assertEqual(1, results['bug_id'])
self.assertEqual(params['test_path'],
json.loads(results['tags'])['test_path'])
self.assertEqual('123', json.loads(results['tags'])['alert'])
def testPinpointParams_InvalidSheriff_RaisesError(self):
params = {'test_path': 'ChromiumPerf/foo/blah/foo'}
with self.assertRaises(pinpoint_request.InvalidParamsError):
pinpoint_request.PinpointParamsFromBisectParams(params)
def testPinpointParams_BotUndefined_ReturnsError(self):
params = {'test_path': 'ChromiumPerf/foo/blah/foo'}
with self.assertRaises(pinpoint_request.InvalidParamsError):
pinpoint_request.PinpointParamsFromBisectParams(params)
