def testStartNewBisectForBug_Pinpoint_ParamsRaisesError(self):
testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'],
{'sunspider': {
'score': {}
}})
test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score')
testing_common.AddRows(
'ChromiumPerf/linux-pinpoint/sunspider/score', {
11999: {
'r_foo': '9e29b5bcd08357155b2859f87227d50ed60cf857'
},
12500: {
'r_foo': 'fc34e5346446854637311ad7793a95d56e314042'
}
})
anomaly.Anomaly(bug_id=333,
test=test_key,
start_revision=12000,
end_revision=12501,
median_before_anomaly=100,
median_after_anomaly=200).put()
result = auto_bisect.StartNewBisectForBug(333)
self.assertEqual({'error': 'Some reason'}, result)
def testPinpointParams_GitHashes(self, mock_commit):
def _MockCommit(git_sha):
if git_sha == 'abc':
return {'number': 1050}
else:
return {'number': 1150}
mock_commit.side_effect = _MockCommit
testing_common.AddTests(['ChromiumPerf'],
['Android Nexus5X WebView Perf'],
{'system_health': {
'foo': {}
}})
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': 'abc',
'end_commit': 'def',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
t = graph_data.TestMetadata(id=params['test_path'])
t.put()
rows = dict(
itertools.chain(
zip(itertools.islice(itertools.count(1000, 2), 50),
itertools.repeat({'value': 0.1})),
zip(itertools.islice(itertools.count(1101, 2), 50),
itertools.repeat({'value': 0.5}))))
testing_common.AddRows(params['test_path'], rows)
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual(0.4, results['comparison_magnitude'])
def testMakeAnomalyEntity_RevisionRanges(self):
testing_common.AddTests(
['ClankInternal'],
['linux'], {
'page_cycler_v2': {
'cnn': {},
'cnn_ref': {},
'yahoo': {},
'nytimes': {},
},
})
test = utils.TestKey('ClankInternal/linux/page_cycler_v2/cnn').get()
testing_common.AddRows(test.test_path, [100, 200, 300, 400])
for row in graph_data.Row.query():
row.r_commit_pos = int(row.value) + 2 # Different enough to ensure it is
# picked up properly.
row.put()
alert = find_anomalies._MakeAnomalyEntity(
_MakeSampleChangePoint(300, 50, 100),
test,
list(graph_data.Row.query()))
self.assertEqual(alert.display_start, 203)
self.assertEqual(alert.display_end, 302)
def testStartNewBisectForBug_WithDefaultRevs_StartsBisect(
self, mock_perform_bisect):
testing_common.AddTests(
['ChromiumPerf'], ['linux-release'], {'sunspider': {'score': {}}})
test_key = utils.TestKey('ChromiumPerf/linux-release/sunspider/score')
testing_common.AddRows(
'ChromiumPerf/linux-release/sunspider/score',
{
11990: {
'a_default_rev': 'r_foo',
'r_foo': '9e29b5bcd08357155b2859f87227d50ed60cf857'
},
12500: {
'a_default_rev': 'r_foo',
'r_foo': 'fc34e5346446854637311ad7793a95d56e314042'
}
})
anomaly.Anomaly(
bug_id=333, test=test_key,
start_revision=12000, end_revision=12500,
median_before_anomaly=100, median_after_anomaly=200).put()
auto_bisect.StartNewBisectForBug(333)
job = try_job.TryJob.query(try_job.TryJob.bug_id == 333).get()
mock_perform_bisect.assert_called_once_with(job)
def _AddData(self):
"""Adds sample TestMetadata entities and returns their keys."""
testing_common.AddTests(['ChromiumPerf'], ['linux'],
{'page_cycler': {
'warm': {
'cnn': {},
}
}})
test_path = 'ChromiumPerf/linux/page_cycler/warm/cnn'
test = utils.TestKey(test_path).get()
test.improvement_direction = anomaly.UP
test.put()
now = datetime.datetime.now()
last_week = now - datetime.timedelta(days=7)
rows = dict([(i * 100, {
'value': i * 1000,
'a_whatever': 'blah',
'r_v8': '1234a',
'timestamp': now if i > 5 else last_week,
'error': 3.3232
}) for i in range(1, 10)])
rows[100]['r_not_every_row'] = 12345
testing_common.AddRows('ChromiumPerf/linux/page_cycler/warm/cnn', rows)
def testPinpointParams_NoAlert(self):
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': '1051',
'end_commit': '1151',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
t = graph_data.TestMetadata(id=params['test_path'])
t.put()
rows = {}
for i in xrange(1000, 1100, 2):
rows[i] = {'value': 0.1}
for i in xrange(1101, 1200, 2):
rows[i] = {'value': 0.5}
testing_common.AddRows(params['test_path'], rows)
results = pinpoint_request.PinpointParamsFromBisectParams(params)
self.assertEqual(0.4, results['comparison_magnitude'])
def _AddTestData(self, series, sheriff_key,
improvement_direction=anomaly.UP):
"""Adds one sample TestMetadata and associated data.
Args:
series: Either a list of values, or a list of (x, y) pairs.
sheriff_key: A Sheriff entity key.
improvement_direction: One of {anomaly.UP, anomaly.DOWN, anomaly.UNKNOWN}.
Returns:
The key of the TestMetadata entity that was added.
"""
testing_common.AddTests(['M'], ['b'], {'benchmark': {'t': {}}})
test_path = 'M/b/benchmark/t'
test = utils.TestKey(test_path).get()
test.improvement_direction = improvement_direction
test.sheriff = sheriff_key
sheriff_entity = sheriff_key.get()
sheriff_entity.patterns.append(test.test_path)
sheriff_entity.put()
if series and isinstance(series[0], (int, float)):
series = enumerate(series, start=1)
testing_common.AddRows(test_path, {x: {'value': y} for x, y in series})
return test.put()
def testPinpointParams_OverriddenAnomalyConfig(self):
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': '1051',
'end_commit': '1151',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
a = anomaly_config.AnomalyConfig()
a.config = {'min_segment_size': 1}
a.patterns = ['*/*/*/*']
a.put()
t = graph_data.TestMetadata(id=params['test_path'])
t.overridden_anomaly_config = a.key
t.put()
rows = dict(
itertools.chain(
zip(
itertools.islice(itertools.count(1000, 2), 75),
itertools.repeat({'value': -100.0})),
[(1050, {'value': 0.1})],
zip(
itertools.islice(itertools.count(1101, 2), 50),
itertools.repeat({'value': 0.5}))))
testing_common.AddRows(params['test_path'], rows)
results = pinpoint_request.PinpointParamsFromBisectParams(params)
# We overrode the anomaly config with a window of 1, and there's only a
# single row with value 0.1, the rest are 0.0.
self.assertEqual(0.4, results['comparison_magnitude'])
def _AddRows(self, keys):
for key in keys:
testing_common.AddRows(utils.TestPath(key), [1, 2, 3, RECENT_REV])
def testStartNewBisectForBut_BlacklistedMaster_SucceedsIfAlternative(
self, _, mock_new):
# Even if there are blacklisted masters, the bisect request should still
# succeed if there's a non-blacklisted master.
namespaced_stored_object.Set('file_bug_bisect_blacklist',
{'ChromiumPerf': []})
namespaced_stored_object.Set('bot_configurations', {
'linux-pinpoint': {
'dimensions': [{
'key': 'foo',
'value': 'bar'
}]
},
})
namespaced_stored_object.Set('repositories', {
'chromium': {
'some': 'params'
},
})
mock_new.return_value = {'jobId': 123, 'jobUrl': 'http://pinpoint/123'}
testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'],
{'sunspider': {
'score': {}
}})
testing_common.AddTests(['ChromiumPerf2'], ['linux-pinpoint'],
{'sunspider': {
'score': {}
}})
test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score')
test_key2 = utils.TestKey(
'ChromiumPerf2/linux-pinpoint/sunspider/score')
rows = {
11999: {
'a_default_rev': 'r_chromium',
'r_chromium': '9e29b5bcd08357155b2859f87227d50ed60cf857'
},
12500: {
'a_default_rev': 'r_chromium',
'r_chromium': 'fc34e5346446854637311ad7793a95d56e314042'
}
}
testing_common.AddRows('ChromiumPerf/linux-pinpoint/sunspider/score',
rows)
testing_common.AddRows('ChromiumPerf2/linux-pinpoint/sunspider/score',
rows)
anomaly.Anomaly(bug_id=333,
test=test_key,
start_revision=12000,
end_revision=12500,
median_before_anomaly=100,
median_after_anomaly=200).put()
a = anomaly.Anomaly(bug_id=333,
test=test_key2,
start_revision=12000,
end_revision=12500,
median_before_anomaly=100,
median_after_anomaly=200).put()
result = auto_bisect.StartNewBisectForBug(333)
self.assertNotIn('error', result)
self.assertEqual({
'alert': a.urlsafe(),
'test_path': test_key2.id()
}, json.loads(mock_new.call_args[0][0]['tags']))
def testMakeAnomalyEntity_AddsOwnership(self):
data_samples = [{
'type': 'GenericSet',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'values': ['*****@*****.**', '*****@*****.**']
}, {
'type': 'GenericSet',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'values': ['abc']
}, {
'type': 'GenericSet',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb828',
'values': ['This is an info blurb.']
}]
test_key = utils.TestKey('ChromiumPerf/linux/page_cycler_v2/cnn')
testing_common.AddTests(
['ChromiumPerf'], ['linux'], {
'page_cycler_v2': {
'cnn': {},
'cnn_ref': {},
'yahoo': {},
'nytimes': {},
},
})
test = test_key.get()
testing_common.AddRows(test.test_path, [100, 200, 300, 400])
suite_key = utils.TestKey('ChromiumPerf/linux/page_cycler_v2')
entity = histogram.SparseDiagnostic(data=data_samples[0],
test=suite_key,
start_revision=1,
end_revision=sys.maxsize,
id=data_samples[0]['guid'],
name=reserved_infos.OWNERS.name)
entity.put()
entity = histogram.SparseDiagnostic(
data=data_samples[1],
test=suite_key,
start_revision=1,
end_revision=sys.maxsize,
id=data_samples[1]['guid'],
name=reserved_infos.BUG_COMPONENTS.name)
entity.put()
entity = histogram.SparseDiagnostic(
data=data_samples[2],
test=suite_key,
start_revision=1,
end_revision=sys.maxsize,
id=data_samples[2]['guid'],
name=reserved_infos.INFO_BLURB.name)
entity.put()
alert = find_anomalies._MakeAnomalyEntity(
_MakeSampleChangePoint(10011, 50, 100), test, 'avg',
self._DataSeries()).get_result()
self.assertEqual(alert.ownership['component'], 'abc')
self.assertListEqual(alert.ownership['emails'],
['*****@*****.**', '*****@*****.**'])
self.assertEqual(alert.ownership['info_blurb'],
'This is an info blurb.')
def _AddRows(self, keys):
for key in keys:
testing_common.AddRows(utils.TestPath(key), [1, 2, 3, 445588])
def testStartNewBisectForBug_Pinpoint_Succeeds(self, mock_new):
namespaced_stored_object.Set('bot_configurations', {
'linux-pinpoint': {
'dimensions': [{
'key': 'foo',
'value': 'bar'
}]
},
})
namespaced_stored_object.Set('repositories', {
'chromium': {
'some': 'params'
},
})
mock_new.return_value = {'jobId': 123, 'jobUrl': 'http://pinpoint/123'}
testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'],
{'sunspider': {
'score': {}
}})
test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score')
testing_common.AddRows(
'ChromiumPerf/linux-pinpoint/sunspider/score', {
11999: {
'a_default_rev': 'r_chromium',
'r_chromium': '9e29b5bcd08357155b2859f87227d50ed60cf857'
},
12500: {
'a_default_rev': 'r_chromium',
'r_chromium': 'fc34e5346446854637311ad7793a95d56e314042'
}
})
t = test_key.get()
t.unescaped_story_name = 'score:foo'
t.put()
a = anomaly.Anomaly(bug_id=333,
project_id='test_project',
test=test_key,
start_revision=12000,
end_revision=12500,
median_before_anomaly=100,
median_after_anomaly=200).put()
result = auto_bisect.StartNewBisectForBug(333, 'test_project')
self.assertEqual({
'issue_id': 123,
'issue_url': 'http://pinpoint/123'
}, result)
self.assertEqual('123', a.get().pinpoint_bisects[0])
self.assertEqual({
'alert': a.urlsafe(),
'test_path': test_key.id()
}, json.loads(mock_new.call_args[0][0]['tags']))
self.assertEqual('score:foo', mock_new.call_args[0][0]['story'])
anomaly_entity = a.get()
anomaly_magnitude = (anomaly_entity.median_after_anomaly -
anomaly_entity.median_before_anomaly)
self.assertEqual(anomaly_magnitude,
mock_new.call_args[0][0]['comparison_magnitude'])
