def testGetTestMetadataKey_TestMetadata(self):
a = anomaly.Anomaly(test=utils.TestKey('a/b/c/d'))
k = a.GetTestMetadataKey()
self.assertEqual('TestMetadata', k.kind())
self.assertEqual('a/b/c/d', k.id())
self.assertEqual('a/b/c/d', utils.TestPath(k))
def testGetTestMetadataKey_None(self):
a = anomaly.Anomaly()
k = a.GetTestMetadataKey()
self.assertIsNone(k)
def testGet_UsesOnlyMostRecentComponents(self):
ownership_samples = [
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'component': 'Abc>Def'
},
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': '123>456'
},
]
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript'])
test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint')
now_datetime = datetime.datetime.now()
older_alert = anomaly.Anomaly(start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[0],
timestamp=now_datetime).put()
newer_alert = anomaly.Anomaly(start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[1],
timestamp=now_datetime +
datetime.timedelta(10)).put()
response = self.testapp.post('/file_bug', [
('keys', '%s,%s' % (older_alert.urlsafe(), newer_alert.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertNotIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="123>456">',
response.body)
response_inverted_order = self.testapp.post('/file_bug', [
('keys', '%s,%s' % (newer_alert.urlsafe(), older_alert.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertNotIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response_inverted_order.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="123>456">',
response_inverted_order.body)
def testGet_UsesFirstDefinedComponent(self):
ownership_samples = [{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': ''
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': 'Abc>Def'
}]
now_datetime = datetime.datetime.now()
test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint')
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript'])
alert_without_ownership = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
timestamp=now_datetime).put()
alert_without_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[0],
timestamp=now_datetime + datetime.timedelta(10)).put()
alert_with_empty_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[1],
timestamp=now_datetime + datetime.timedelta(20)).put()
alert_with_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[2],
timestamp=now_datetime + datetime.timedelta(30)).put()
response = self.testapp.post('/file_bug', [
('keys', '%s,%s,%s,%s' % (alert_without_ownership.urlsafe(),
alert_without_component.urlsafe(),
alert_with_empty_component.urlsafe(),
alert_with_component.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response.body)
def _MakeAnomalyEntity(change_point, test, rows):
"""Creates an Anomaly entity.
Args:
change_point: A find_change_points.ChangePoint object.
test: The TestMetadata entity that the anomalies were found on.
rows: List of Row entities that the anomalies were found on.
Returns:
An Anomaly entity, which is not yet put in the datastore.
"""
end_rev = change_point.x_value
start_rev = _GetImmediatelyPreviousRevisionNumber(end_rev, rows) + 1
display_start = display_end = None
if test.master_name == 'ClankInternal':
display_start, display_end = _GetDisplayRange(change_point.x_value,
rows)
median_before = change_point.median_before
median_after = change_point.median_after
suite_key = test.key.id().split('/')[:3]
suite_key = '/'.join(suite_key)
suite_key = utils.TestKey(suite_key)
queried_diagnostics = yield (
histogram.SparseDiagnostic.GetMostRecentDataByNamesAsync(
suite_key,
set([
reserved_infos.BUG_COMPONENTS.name, reserved_infos.OWNERS.name
])))
bug_components = queried_diagnostics.get(
reserved_infos.BUG_COMPONENTS.name, {}).get('values')
ownership_information = {
'emails':
queried_diagnostics.get(reserved_infos.OWNERS.name, {}).get('values'),
'component': (bug_components[0] if bug_components else None)
}
new_anomaly = anomaly.Anomaly(
start_revision=start_rev,
end_revision=end_rev,
median_before_anomaly=median_before,
median_after_anomaly=median_after,
segment_size_before=change_point.size_before,
segment_size_after=change_point.size_after,
window_end_revision=change_point.window_end,
std_dev_before_anomaly=change_point.std_dev_before,
t_statistic=change_point.t_statistic,
degrees_of_freedom=change_point.degrees_of_freedom,
p_value=change_point.p_value,
is_improvement=_IsImprovement(test, median_before, median_after),
ref_test=_GetRefBuildKeyForTest(test),
test=test.key,
sheriff=test.sheriff,
internal_only=test.internal_only,
units=test.units,
display_start=display_start,
display_end=display_end,
ownership=ownership_information)
raise ndb.Return(new_anomaly)
def testGetGraphJson_WithAnomalies_ReturnsCorrectAnomalyAnnotations(self):
self._AddTestColumns()
anomaly1 = anomaly.Anomaly(
start_revision=14999, end_revision=15000,
test=utils.TestKey('ChromiumGPU/win7/dromaeo/dom'),
median_before_anomaly=100,
median_after_anomaly=200)
anomaly1.SetIsImprovement()
key1 = anomaly1.put()
anomaly2 = anomaly.Anomaly(
start_revision=15004, end_revision=15006,
test=utils.TestKey('ChromiumGPU/win7/dromaeo/dom'),
median_before_anomaly=200,
median_after_anomaly=100,
bug_id=12345)
anomaly2.SetIsImprovement()
key2 = anomaly2.put()
test = utils.TestKey('ChromiumGPU/win7/dromaeo/dom').get()
test.description = 'About this test'
test.units = 'ms'
test.buildername = 'Windows 7 (1)'
test.put()
flot_json_str = graph_json.GetGraphJson(
{
'ChromiumGPU/win7/dromaeo/dom': [],
},
rev=15000, num_points=8)
flot = json.loads(flot_json_str)
annotations = flot['annotations']
self.assertEqual(5, len(annotations['0']))
# Verify key fields of the annotation dictionary for the first anomaly.
anomaly_one_annotation = annotations['0']['0']['g_anomaly']
self.assertEqual(14999, anomaly_one_annotation['start_revision'])
self.assertEqual(15000, anomaly_one_annotation['end_revision'])
self.assertEqual('100.0%', anomaly_one_annotation['percent_changed'])
self.assertIsNone(anomaly_one_annotation['bug_id'])
self.assertEqual(key1.urlsafe(), anomaly_one_annotation['key'])
self.assertTrue(anomaly_one_annotation['improvement'])
# Verify key fields of the annotation dictionary for th second anomaly.
anomaly_two_annotation = annotations['0']['2']['g_anomaly']
self.assertEqual(15004, anomaly_two_annotation['start_revision'])
self.assertEqual(15006, anomaly_two_annotation['end_revision'])
self.assertEqual('50.0%', anomaly_two_annotation['percent_changed'])
self.assertEqual(12345, anomaly_two_annotation['bug_id'])
self.assertEqual(key2.urlsafe(), anomaly_two_annotation['key'])
self.assertFalse(anomaly_two_annotation['improvement'])
# Verify the tracing link annotations
self.assertEqual('http://trace/15000',
annotations['0']['0']['a_tracing_uri'])
self.assertEqual('http://trace/15012',
annotations['0']['4']['a_tracing_uri'])
# Verify the tracing rerun options
self.assertEqual({'foo': '--foo'},
annotations['0']['0']['a_trace_rerun_options'])
# Verify the series annotations.
self.assertEqual({
'0': {
'name': 'dom',
'path': 'ChromiumGPU/win7/dromaeo/dom',
'units': 'ms',
'better': 'Higher',
'description': 'About this test'
}
}, annotations['series'])
def _AddMockAlertSummaryData(self):
"""Adds data to be used in the alert-summary stats tests below."""
correct_sheriff = sheriff.Sheriff(
id='Chromium Perf Sheriff', email='*****@*****.**', patterns=[]).put()
wrong_sheriff = sheriff.Sheriff(
id='Some other sheriff', email='*****@*****.**', patterns=[]).put()
linux_sunspider = 'ChromiumPerf/linux-release/sunspider/Total'
linux_octane = 'ChromiumPerf/linux-release/octane/Total'
linux_media = 'ChromiumPerf/linux-release/media.tough_media_cases/Total'
windows_sunspider = 'ChromiumPerf/windows/sunspider/Total'
windows_octane = 'ChromiumPerf/windows/octane/Total'
# Should not be included: too early.
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 1),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
# Should not be included: too late.
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 6, 17),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
# Should not be included: wrong sheriff.
anomaly.Anomaly(
sheriff=wrong_sheriff,
timestamp=datetime.datetime(2013, 5, 11),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
# Everything below should be included.
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(linux_octane),
median_before_anomaly=100,
median_after_anomaly=101.5,
bug_id=-1).put()
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(linux_media),
median_before_anomaly=100,
median_after_anomaly=101.5,
bug_id=-2).put()
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(windows_sunspider),
median_before_anomaly=100,
median_after_anomaly=104.5,
bug_id=12345).put()
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(windows_octane),
median_before_anomaly=100,
median_after_anomaly=600,
bug_id=12345).put()
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 15),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=200).put()
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 20),
test=utils.TestKey(windows_sunspider),
median_before_anomaly=100,
median_after_anomaly=115,
bug_id=12345).put()
anomaly.Anomaly(
sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 21),
test=utils.TestKey(windows_octane),
median_before_anomaly=100,
median_after_anomaly=104).put()
def testGet_ComponentsChosenPerTest(self):
ownership_samples = [
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'component': 'Abc>Def'
},
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': '123>456'
},
]
sheriff_key = sheriff.Sheriff(
id='Sheriff',
labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put()
test_paths = [
'ChromiumPerf/linux/scrolling/first_paint',
'ChromiumPerf/linux/scrolling/mean_frame_time'
]
test_keys = [utils.TestKey(test_path) for test_path in test_paths]
now_datetime = datetime.datetime.now()
alert_test_key_0 = anomaly.Anomaly(start_revision=1476193320,
end_revision=1476201870,
test=test_keys[0],
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
ownership=ownership_samples[0],
timestamp=now_datetime).put()
alert_test_key_1 = anomaly.Anomaly(start_revision=1476193320,
end_revision=1476201870,
test=test_keys[1],
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
ownership=ownership_samples[1],
timestamp=now_datetime +
datetime.timedelta(10)).put()
response = self.testapp.post('/file_bug', [
('keys', '%s,%s' %
(alert_test_key_0.urlsafe(), alert_test_key_1.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="123>456">',
response.body)
def testGet_WithAllOwnershipComponents(self):
ownership_samples = [{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': 'Abc>Xyz'
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'component': 'Def>123'
}]
test_paths = [
'ChromiumPerf/linux/scrolling/first_paint',
'ChromiumPerf/linux/scrolling/mean_frame_time'
]
test_keys = [utils.TestKey(test_path) for test_path in test_paths]
sheriff_key = sheriff.Sheriff(
id='Sheriff',
labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put()
anomaly_1 = anomaly.Anomaly(start_revision=1476193324,
end_revision=1476201840,
test=test_keys[0],
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
ownership=ownership_samples[0]).put()
anomaly_2 = anomaly.Anomaly(start_revision=1476193320,
end_revision=1476201870,
test=test_keys[1],
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
ownership=ownership_samples[1]).put()
response = self.testapp.post('/file_bug', [
('keys', '%s' % (anomaly_1.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Xyz">',
response.body)
response_with_both_anomalies = self.testapp.post(
'/file_bug', [
('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Xyz">',
response_with_both_anomalies.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="Def>123">',
response_with_both_anomalies.body)
def _AddAlertsToDataStore(self):
"""Adds sample data, including triaged and non-triaged alerts."""
key_map = {}
sheriff_key = sheriff.Sheriff(id='Chromium Perf Sheriff',
email='*****@*****.**').put()
testing_common.AddTests(
['ChromiumGPU'], ['linux-release'], {
'scrolling-benchmark': {
'first_paint': {},
'first_paint_ref': {},
'mean_frame_time': {},
'mean_frame_time_ref': {},
}
})
first_paint = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/first_paint')
mean_frame_time = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time')
# By default, all TestMetadata entities have an improvement_direction of
# UNKNOWN, meaning that neither direction is considered an improvement.
# Here we set the improvement direction so that some anomalies are
# considered improvements.
for test_key in [first_paint, mean_frame_time]:
test = test_key.get()
test.improvement_direction = anomaly.DOWN
test.put()
# Add some (12) non-triaged alerts.
for end_rev in range(10000, 10120, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
ref_test=ref_test_key,
sheriff=sheriff_key)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
# Add some (2) already-triaged alerts.
for end_rev in range(10120, 10140, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
bug_id = -1 if end_rev % 20 == 0 else 12345
anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
ref_test=ref_test_key,
bug_id=bug_id,
sheriff=sheriff_key)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
if bug_id > 0:
bug_data.Bug(id=bug_id).put()
# Add some (6) non-triaged improvements.
for end_rev in range(10140, 10200, 10):
test_key = mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=200,
median_after_anomaly=100,
ref_test=ref_test_key,
sheriff=sheriff_key)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
self.assertTrue(anomaly_entity.is_improvement)
key_map[end_rev] = anomaly_key.urlsafe()
return key_map
def testStartNewBisectForBut_BlacklistedMaster_SucceedsIfAlternative(
self, mock_new):
# Even if there are blacklisted masters, the bisect request should still
# succeed if there's a non-blacklisted master.
namespaced_stored_object.Set('file_bug_bisect_blacklist',
{'ChromiumPerf': []})
namespaced_stored_object.Set('bot_configurations', {
'linux-pinpoint': {
'dimensions': [{
'key': 'foo',
'value': 'bar'
}]
},
})
namespaced_stored_object.Set('repositories', {
'chromium': {
'some': 'params'
},
})
mock_new.return_value = {'jobId': 123, 'jobUrl': 'http://pinpoint/123'}
testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'],
{'sunspider': {
'score': {}
}})
testing_common.AddTests(['ChromiumPerf2'], ['linux-pinpoint'],
{'sunspider': {
'score': {}
}})
test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score')
test_key2 = utils.TestKey(
'ChromiumPerf2/linux-pinpoint/sunspider/score')
rows = {
11999: {
'a_default_rev': 'r_chromium',
'r_chromium': '9e29b5bcd08357155b2859f87227d50ed60cf857'
},
12500: {
'a_default_rev': 'r_chromium',
'r_chromium': 'fc34e5346446854637311ad7793a95d56e314042'
}
}
testing_common.AddRows('ChromiumPerf/linux-pinpoint/sunspider/score',
rows)
testing_common.AddRows('ChromiumPerf2/linux-pinpoint/sunspider/score',
rows)
anomaly.Anomaly(bug_id=333,
test=test_key,
start_revision=12000,
end_revision=12500,
median_before_anomaly=100,
median_after_anomaly=200).put()
a = anomaly.Anomaly(bug_id=333,
test=test_key2,
start_revision=12000,
end_revision=12500,
median_before_anomaly=100,
median_after_anomaly=200).put()
result = auto_bisect.StartNewBisectForBug(333)
self.assertNotIn('error', result)
self.assertEqual({
'alert': a.urlsafe(),
'test_path': test_key2.id()
}, json.loads(mock_new.call_args[0][0]['tags']))
