def _GetDefaultMailArgs(self):
"""Adds an Anomaly and returns arguments for email_sheriff.EmailSheriff."""
test_entity = self._AddTestToStubDataStore()
subscription_url = Subscription(
name='Chromium Perf Sheriff URL',
rotation_url=_SHERIFF_URL,
bug_labels=['Performance-Sheriff-URL']
)
subscription_email = Subscription(
name='Chromium Perf Sheriff Mail',
notification_email=_SHERIFF_EMAIL,
bug_labels=['Performance-Sheriff-Mail']
)
anomaly_entity = anomaly.Anomaly(
median_before_anomaly=5.0,
median_after_anomaly=10.0,
start_revision=10002,
end_revision=10004,
subscription_names=[
subscription_url.name,
subscription_email.name,
],
subscriptions=[subscription_url, subscription_email],
test=utils.TestKey('ChromiumPerf/Win7/dromaeo/dom'))
return {
'subscriptions': [subscription_url, subscription_email],
'test': test_entity,
'anomaly': anomaly_entity
}
def testPost_SheriffParameterSet_OtherSheriffAlertsListed(self):
self._AddAlertsToDataStore()
subscription = Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
mean_frame_time = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time')
anomalies, _, _ = anomaly.Anomaly.QueryAsync(
test=mean_frame_time).get_result()
for anomaly_entity in anomalies:
anomaly_entity.subscriptions = [subscription]
anomaly_entity.subscription_names = [subscription.name]
anomaly_entity.put()
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
),
Subscription(
name='Sheriff2',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts', {'sheriff': 'Sheriff2'})
anomaly_list = self.GetJsonValue(response, 'anomaly_list')
sheriff_list = self.GetJsonValue(response, 'sheriff_list')
for alert in anomaly_list:
self.assertEqual('mean_frame_time', alert['test'])
self.assertEqual(2, len(sheriff_list))
self.assertEqual('Chromium Perf Sheriff', sheriff_list[0])
self.assertEqual('Sheriff2', sheriff_list[1])
def testGet_DumpAnomaliesDataForSheriff(self):
# Insert some test, sheriffs and alerts.
testing_common.AddTests('M', 'b', {'foo': {}})
testing_common.AddTests('M', 'b', {'bar': {}})
test_key_foo = utils.TestKey('M/b/foo')
test_key_bar = utils.TestKey('M/b/bar')
test_con_foo_key = utils.GetTestContainerKey(test_key_foo)
test_con_bar_key = utils.GetTestContainerKey(test_key_bar)
chromium_subscription = Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**'
)
qa_subscription = Subscription(
name='QA Perf Sheriff',
notification_email='*****@*****.**'
)
anomaly.Anomaly(
subscriptions=[chromium_subscription],
subscription_names=[chromium_subscription.name],
test=test_key_foo).put()
anomaly.Anomaly(
subscriptions=[qa_subscription],
subscription_names=[qa_subscription.name],
test=test_key_bar).put()
default_max_points = dump_graph_json._DEFAULT_MAX_POINTS
# Add some rows.
rows = []
for rev in range(1, default_max_points * 2):
row = graph_data.Row(parent=test_con_foo_key, id=rev, value=(rev * 2))
rows.append(row)
row = graph_data.Row(parent=test_con_bar_key, id=rev, value=(rev * 2))
rows.append(row)
ndb.put_multi(rows)
# Anomaly entities, Row entities, TestMetadata, and Sheriff entities for
# parameter 'sheriff' should be returned.
response = self.testapp.get(
'/dump_graph_json',
{'sheriff': 'Chromium Perf Sheriff'})
protobuf_strings = json.loads(response.body)
self.assertEqual(default_max_points + 5, len(protobuf_strings))
entities = list(
map(dump_graph_json.BinaryProtobufToEntity, protobuf_strings))
rows = _EntitiesOfKind(entities, 'Row')
anomalies = _EntitiesOfKind(entities, 'Anomaly')
subscriptions = _EntitiesOfKind(entities, 'Subscription')
self.assertEqual(default_max_points, len(rows))
self.assertEqual(1, len(anomalies))
self.assertEqual(1, len(subscriptions))
self.assertEqual('Chromium Perf Sheriff', subscriptions[0].name)
def _AddSampleAlerts(self,
master='ChromiumPerf',
has_commit_positions=True):
"""Adds sample data and returns a dict of rev to anomaly key."""
# Add sample sheriff, masters, bots, and tests.
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff'],
bug_components=['Blink>Javascript'],
)
testing_common.AddTests(
[master], ['linux'],
{'scrolling': {
'first_paint': {},
'mean_frame_time': {},
}})
test_path1 = '%s/linux/scrolling/first_paint' % master
test_path2 = '%s/linux/scrolling/mean_frame_time' % master
test_key1 = utils.TestKey(test_path1)
test_key2 = utils.TestKey(test_path2)
anomaly_key1 = self._AddAnomaly(111995, 112005, test_key1,
subscription)
anomaly_key2 = self._AddAnomaly(112000, 112010, test_key2,
subscription)
anomaly_key3 = self._AddAnomaly(112015, 112015, test_key2,
subscription)
rows_1 = testing_common.AddRows(test_path1, [112005])
rows_2 = testing_common.AddRows(test_path2, [112010])
rows_2 = testing_common.AddRows(test_path2, [112015])
if has_commit_positions:
rows_1[0].r_commit_pos = 112005
rows_2[0].r_commit_pos = 112010
return (anomaly_key1, anomaly_key2, anomaly_key3)
def _CreateAnomaly(self,
timestamp=None,
bug_id=None,
sheriff_name=None,
test='master/bot/test_suite/measurement/test_case',
start_revision=0,
end_revision=100,
display_start=0,
display_end=100,
median_before_anomaly=100,
median_after_anomaly=200,
is_improvement=False,
recovered=False):
entity = anomaly.Anomaly()
if timestamp:
entity.timestamp = timestamp
entity.bug_id = bug_id
if sheriff_name:
entity.subscription_names.append(sheriff_name)
entity.subscriptions.append(
Subscription(name=sheriff_name,
notification_email='*****@*****.**'))
if test:
entity.test = utils.TestKey(test)
entity.start_revision = start_revision
entity.end_revision = end_revision
entity.display_start = display_start
entity.display_end = display_end
entity.median_before_anomaly = median_before_anomaly
entity.median_after_anomaly = median_after_anomaly
entity.is_improvement = is_improvement
entity.recovered = recovered
return entity.put().urlsafe()
def testGet_WithAllOwnershipComponents(self):
ownership_samples = [{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': 'Abc>Xyz'
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'component': 'Def>123'
}]
test_paths = [
'ChromiumPerf/linux/scrolling/first_paint',
'ChromiumPerf/linux/scrolling/mean_frame_time'
]
test_keys = [utils.TestKey(test_path) for test_path in test_paths]
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript'])
anomaly_1 = anomaly.Anomaly(start_revision=1476193324,
end_revision=1476201840,
test=test_keys[0],
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[0]).put()
anomaly_2 = anomaly.Anomaly(start_revision=1476193320,
end_revision=1476201870,
test=test_keys[1],
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[1]).put()
response = self.testapp.post('/file_bug', [
('keys', '%s' % (anomaly_1.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Xyz">',
response.body)
response_with_both_anomalies = self.testapp.post(
'/file_bug', [
('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Xyz">',
response_with_both_anomalies.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="Def>123">',
response_with_both_anomalies.body)
def testProcessTest_InternalOnlyTest(self, mock_email_sheriff):
self._AddDataForTests()
test = utils.TestKey(
'ChromiumGPU/linux-release/scrolling_benchmark/ref').get()
test.internal_only = True
test.UpdateSheriff()
test.put()
s = Subscription(name='sheriff', visibility=VISIBILITY.PUBLIC)
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([s], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.key.id())])
self.ExecuteDeferredTasks('default')
expected_calls = [
mock.call([ModelMatcher('sheriff')],
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10011))
]
self.assertEqual(expected_calls, mock_email_sheriff.call_args_list)
anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(len(anomalies), 1)
self.assertEqual(test.key, anomalies[0].test)
self.assertEqual(100, anomalies[0].percent_changed)
self.assertEqual(anomaly.UP, anomalies[0].direction)
self.assertEqual(10007, anomalies[0].start_revision)
self.assertEqual(10011, anomalies[0].end_revision)
self.assertTrue(anomalies[0].internal_only)
def testPost_SheriffParameterSet_OtherSheriffAlertsListed(self):
self._AddAlertsToDataStore()
# We still need sheriff information from here before sheriff-config
# provide a way to fetch subsciber list.
sheriff.Sheriff(
id='Sheriff2', email='*****@*****.**').put()
subscription = Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
mean_frame_time = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time')
anomalies, _, _ = anomaly.Anomaly.QueryAsync(
test=mean_frame_time).get_result()
for anomaly_entity in anomalies:
anomaly_entity.subscriptions = [subscription]
anomaly_entity.subscription_names = [subscription.name]
anomaly_entity.put()
response = self.testapp.post('/alerts', {'sheriff': 'Sheriff2'})
anomaly_list = self.GetJsonValue(response, 'anomaly_list')
sheriff_list = self.GetJsonValue(response, 'sheriff_list')
for alert in anomaly_list:
self.assertEqual('mean_frame_time', alert['test'])
self.assertEqual(2, len(sheriff_list))
self.assertEqual('Chromium Perf Sheriff', sheriff_list[0])
self.assertEqual('Sheriff2', sheriff_list[1])
def testGet_OwnersNotFilledWhenNoOwnership(self):
test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint')
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript'])
anomaly_entity = anomaly.Anomaly(
start_revision=1476193324,
end_revision=1476201840,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
).put()
response = self.testapp.post('/file_bug', [
('keys', '%s' % (anomaly_entity.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn('<input type="text" name="owner" value="">',
response.body)
def testPost_TriagedParameterSet_TriagedListed(self):
self._AddAlertsToDataStore()
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts', {'triaged': 'true'})
anomaly_list = self.GetJsonValue(response, 'anomaly_list')
# The alerts listed should contain those added above, including alerts
# that have a bug ID that is not None.
self.assertEqual(14, len(anomaly_list))
expected_end_rev = 10130
# The test below depends on the order of the items, but the order is not
# guaranteed; it depends on the timestamps, which depend on put order.
anomaly_list.sort(key=lambda a: -a['end_revision'])
for alert in anomaly_list:
if expected_end_rev == 10130:
self.assertEqual(12345, alert['bug_id'])
elif expected_end_rev == 10120:
self.assertEqual(-1, alert['bug_id'])
else:
self.assertIsNone(alert['bug_id'])
expected_end_rev -= 10
self.assertEqual(expected_end_rev, 9990)
def testGet_ComponentsChosenPerTest(self):
ownership_samples = [
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'component': 'Abc>Def'
},
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': '123>456'
},
]
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript'])
test_paths = [
'ChromiumPerf/linux/scrolling/first_paint',
'ChromiumPerf/linux/scrolling/mean_frame_time'
]
test_keys = [utils.TestKey(test_path) for test_path in test_paths]
now_datetime = datetime.datetime.now()
alert_test_key_0 = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_keys[0],
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[0],
timestamp=now_datetime).put()
alert_test_key_1 = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_keys[1],
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[1],
timestamp=now_datetime + datetime.timedelta(10)).put()
response = self.testapp.post('/file_bug', [
('keys', '%s,%s' %
(alert_test_key_0.urlsafe(), alert_test_key_1.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="123>456">',
response.body)
def _MockData(self, path='master/bot/suite/measure/case',
internal_only=False):
test = graph_data.TestMetadata(
has_rows=True,
id=path,
improvement_direction=anomaly.DOWN,
internal_only=internal_only,
units='units')
test.UpdateSheriff()
test.put()
for i in range(1, 21, 2):
graph_data.Row(
error=(i / 2.0),
id=i,
parent=test.key,
r_i2=(i * 2),
timestamp=datetime.datetime.utcfromtimestamp(i),
value=float(i)).put()
histogram.Histogram(
data=_TEST_HISTOGRAM_DATA,
id=str(uuid.uuid4()),
internal_only=internal_only,
revision=i,
test=test.key).put()
anomaly.Anomaly(
end_revision=11,
internal_only=internal_only,
is_improvement=False,
median_after_anomaly=6,
median_before_anomaly=4,
subscriptions=[Subscription(
name='Taylor',
notification_email=testing_common.INTERNAL_USER.email(),
)],
subscription_names=['Taylor'],
start_revision=10,
test=test.key).put()
histogram.SparseDiagnostic(
data={'type': 'GenericSet', 'guid': str(uuid.uuid4()), 'values': [1]},
end_revision=11,
id=str(uuid.uuid4()),
internal_only=internal_only,
name=reserved_infos.DEVICE_IDS.name,
start_revision=1,
test=test.key).put()
histogram.SparseDiagnostic(
data={'type': 'GenericSet', 'guid': str(uuid.uuid4()), 'values': [2]},
end_revision=None,
id=str(uuid.uuid4()),
internal_only=internal_only,
name=reserved_infos.DEVICE_IDS.name,
start_revision=11,
test=test.key).put()
def testGet_OwnersAreEmptyEvenWithOwnership(self):
ownership_samples = [{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'emails': ['*****@*****.**']
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'emails': ['*****@*****.**']
}]
test_paths = [
'ChromiumPerf/linux/scrolling/first_paint',
'ChromiumPerf/linux/scrolling/mean_frame_time'
]
test_keys = [utils.TestKey(test_path) for test_path in test_paths]
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript'])
anomaly_1 = anomaly.Anomaly(
start_revision=1476193324,
end_revision=1476201840,
test=test_keys[0],
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[0]).put()
anomaly_2 = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_keys[1],
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[1]).put()
response = self.testapp.post('/file_bug', [
('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn('<input type="text" name="owner" value="">', response.body)
response_changed_order = self.testapp.post('/file_bug', [
('keys', '%s,%s' % (anomaly_2.urlsafe(), anomaly_1.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn('<input type="text" name="owner" value="">',
response_changed_order.body)
def _ParseSubscription(revision, subscription):
return Subscription(
revision=revision,
name=subscription.name,
rotation_url=subscription.rotation_url,
notification_email=subscription.notification_email,
bug_labels=list(subscription.bug_labels),
bug_components=list(subscription.bug_components),
bug_cc_emails=list(subscription.bug_cc_emails),
visibility=subscription.visibility,
)
def testPost_ImprovementsParameterSet_ListsImprovements(self):
self._AddAlertsToDataStore()
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts', {'improvements': 'true'})
anomaly_list = self.GetJsonValue(response, 'anomaly_list')
self.assertEqual(18, len(anomaly_list))
def testPost_AnomalyCursorSet_ReturnsNextCursorAndShowMore(self):
self._AddAlertsToDataStore()
# Need to post to the app once to get the initial cursor.
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts',
{'max_anomalies_to_show': 5})
anomaly_list = self.GetJsonValue(response, 'anomaly_list')
anomaly_cursor = self.GetJsonValue(response, 'anomaly_cursor')
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts', {
'anomaly_cursor': anomaly_cursor,
'max_anomalies_to_show': 5
})
anomaly_list2 = self.GetJsonValue(response, 'anomaly_list')
anomalies_show_more = self.GetJsonValue(response,
'show_more_anomalies')
anomaly_cursor = self.GetJsonValue(response, 'anomaly_cursor')
anomaly_count = self.GetJsonValue(response, 'anomaly_count')
self.assertEqual(5, len(anomaly_list2))
self.assertTrue(anomalies_show_more)
self.assertIsNotNone(anomaly_cursor) # Don't know what this will be.
self.assertEqual(12, anomaly_count)
for a in anomaly_list: # Ensure anomaly_lists aren't equal.
self.assertNotIn(a, anomaly_list2)
def testPost_ExternalUserRequestsInternalOnlySheriff_ErrorMessage(self):
self.UnsetCurrentUser()
self.assertFalse(utils.IsInternalUser())
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts?sheriff=Foo')
error = self.GetJsonValue(response, 'error')
self.assertIsNotNone(error)
def testMatch(self):
clt = sheriff_config_client.SheriffConfigClient()
response_text = """
{
"subscriptions": [
{
"config_set": "projects/catapult",
"revision": "c9d4943dc832e448f9786e244f918fdabc1e5303",
"subscription": {
"name": "Public Team1",
"rotation_url": "https://some/url",
"notification_email": "*****@*****.**",
"monorail_project_id": "non-chromium",
"bug_labels": [
"Lable1",
"Lable2"
],
"bug_components": [
"foo>bar"
],
"visibility": "PUBLIC",
"patterns": [
{
"glob": "Foo2/*/Bar2/*"
},
{
"regex": ".*"
}
]
}
}
]
}
"""
clt._session = self._Session(self._Response(True, response_text))
expected = [
Subscription(
revision='c9d4943dc832e448f9786e244f918fdabc1e5303',
name='Public Team1',
rotation_url='https://some/url',
notification_email='*****@*****.**',
visibility=VISIBILITY.PUBLIC,
bug_labels=['Lable1', 'Lable2'],
bug_components=['foo>bar'],
auto_triage_enable=False,
auto_bisect_enable=False,
monorail_project_id='non-chromium',
),
]
self.assertEqual(clt.Match('Foo2/a/Bar2/b'), (expected, None))
def testProcessTest_FiltersOutImprovements(self, mock_email_sheriff):
self._AddDataForTests()
test = utils.TestKey(
'ChromiumGPU/linux-release/scrolling_benchmark/ref').get()
test.improvement_direction = anomaly.UP
test.UpdateSheriff()
test.put()
s = Subscription(name='sheriff', visibility=VISIBILITY.PUBLIC)
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([s], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.key.id())])
self.ExecuteDeferredTasks('default')
mock_email_sheriff.assert_called_once_with(
[ModelMatcher('sheriff')],
ModelMatcher('ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10041))
def testSupportsCCList(self):
http = utils.ServiceAccountHttp()
owner = ''
cc = '[email protected],[email protected],[email protected],,'
summary = 'test'
description = 'Test test.'
labels = []
components = []
test_path = 'ChromiumPerf/linux/scrolling/first_paint'
test_key = utils.TestKey(test_path)
subscription = Subscription(name='Sheriff', )
keys = [self._AddAnomaly(10, 20, test_key, subscription).urlsafe()]
bisect = False
result = file_bug.FileBug(http, owner, cc, summary, description,
labels, components, keys, bisect)
self.assertNotIn('bisect_error', result)
self.assertNotIn('jobId', result)
def _AddAnomalies(self):
"""Adds sample Anomaly data and returns a dict of revision to key."""
subscription = Subscription(
name='Chromium Perf Sheriff', notification_email='*****@*****.**')
test_keys = self._AddTests()
key_map = {}
# Add anomalies to the two tests alternately.
for end_rev in range(10000, 10120, 10):
test_key = test_keys[0] if end_rev % 20 == 0 else test_keys[1]
anomaly_key = anomaly.Anomaly(
start_revision=(end_rev - 5),
end_revision=end_rev,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
).put()
key_map[end_rev] = anomaly_key.urlsafe()
# Add an anomaly that overlaps.
anomaly_key = anomaly.Anomaly(
start_revision=9990,
end_revision=9996,
test=test_keys[0],
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
).put()
key_map[9996] = anomaly_key.urlsafe()
# Add an anomaly that overlaps and has bug ID.
anomaly_key = anomaly.Anomaly(
start_revision=9990,
end_revision=9997,
test=test_keys[0],
median_before_anomaly=100,
median_after_anomaly=200,
bug_id=12345,
subscriptions=[subscription],
subscription_names=[subscription.name],
).put()
key_map[9997] = anomaly_key.urlsafe()
return key_map
def testV2(self):
alert = anomaly.Anomaly(
bug_id=10,
end_revision=20,
internal_only=True,
is_improvement=True,
median_after_anomaly=30,
median_before_anomaly=40,
recovered=True,
subscription_names=['Sheriff2'],
subscriptions=[
Subscription(name='Sheriff2',
bug_components=['component'],
notification_email='*****@*****.**')
],
start_revision=5,
test=utils.TestKey('m/b/s/m/c'),
units='ms',
).put().get()
actual = alerts.GetAnomalyDict(alert, v2=True)
del actual['dashboard_link']
self.assertEqual(
{
'bug_components': ['component'],
'bug_id': 10,
'project_id': 'chromium',
'bug_labels': ['Restrict-View-Google'],
'descriptor': {
'testSuite': 's',
'measurement': 'm',
'bot': 'm:b',
'testCase': 'c',
'statistic': None,
},
'end_revision': 20,
'improvement': True,
'key': alert.key.urlsafe(),
'median_after_anomaly': 30,
'median_before_anomaly': 40,
'recovered': True,
'start_revision': 5,
'units': 'ms',
'pinpoint_bisects': [],
}, actual)
def testDumpJsonWithAlertData(self):
testing_common.AddTests('M', 'b', {'foo': {}})
test_key = utils.TestKey('M/b/foo')
subscription = Subscription(notification_email='*****@*****.**')
anomaly.Anomaly(subscriptions=[subscription], test=test_key).put()
# Anomaly entities for the requested test, as well as sheriffs for
# the aforementioned Anomaly, should be returned.
response = self.testapp.get('/dump_graph_json',
{'test_path': 'M/b/foo'})
protobuf_strings = json.loads(response.body)
self.assertEqual(5, len(protobuf_strings))
entities = list(
map(dump_graph_json.BinaryProtobufToEntity, protobuf_strings))
anomalies = _EntitiesOfKind(entities, 'Anomaly')
subscriptions = _EntitiesOfKind(entities, 'Subscription')
self.assertEqual(1, len(anomalies))
self.assertEqual(1, len(subscriptions))
self.assertEqual('*****@*****.**',
subscriptions[0].notification_email)
def testPost_NoParametersSet_UntriagedAlertsListed(self):
key_map = self._AddAlertsToDataStore()
with mock.patch.object(
SheriffConfigClient, 'List',
mock.MagicMock(return_value=([
Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
], None))):
response = self.testapp.post('/alerts')
anomaly_list = self.GetJsonValue(response, 'anomaly_list')
self.assertEqual(12, len(anomaly_list))
# The test below depends on the order of the items, but the order is not
# guaranteed; it depends on the timestamps, which depend on put order.
anomaly_list.sort(key=lambda a: -a['end_revision'])
expected_end_rev = 10110
for alert in anomaly_list:
self.assertEqual(expected_end_rev, alert['end_revision'])
self.assertEqual(expected_end_rev - 5, alert['start_revision'])
self.assertEqual(key_map[expected_end_rev], alert['key'])
self.assertEqual('ChromiumGPU', alert['master'])
self.assertEqual('linux-release', alert['bot'])
self.assertEqual('scrolling-benchmark', alert['testsuite'])
if expected_end_rev % 20 == 0:
self.assertEqual('first_paint', alert['test'])
self.assertEqual(
'ChromiumGPU/linux-release/scrolling-benchmark/first_paint_ref',
alert['ref_test'])
else:
self.assertEqual('mean_frame_time', alert['test'])
self.assertEqual(
'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time_ref',
alert['ref_test'])
self.assertEqual('100.0%', alert['percent_changed'])
self.assertIsNone(alert['bug_id'])
expected_end_rev -= 10
self.assertEqual(expected_end_rev, 9990)
def _AddSampleClankAlerts(self):
"""Adds sample data and returns a dict of rev to anomaly key.
The biggest difference here is that the start/end revs aren't chromium
commit positions. This tests the _MilestoneLabel function to make sure
it will update the end_revision if r_commit_pos is found.
"""
# Add sample sheriff, masters, bots, and tests. Doesn't need to be Clank.
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff'],
bug_components=['Blink>Javascript'],
)
testing_common.AddTests(
['ChromiumPerf'], ['linux'],
{'scrolling': {
'first_paint': {},
'mean_frame_time': {},
}})
test_path1 = 'ChromiumPerf/linux/scrolling/first_paint'
test_path2 = 'ChromiumPerf/linux/scrolling/mean_frame_time'
test_key1 = utils.TestKey(test_path1)
test_key2 = utils.TestKey(test_path2)
anomaly_key1 = self._AddAnomaly(1476193324, 1476201840, test_key1,
subscription)
anomaly_key2 = self._AddAnomaly(1476193320, 1476201870, test_key2,
subscription)
anomaly_key3 = self._AddAnomaly(1476193390, 1476193390, test_key2,
subscription)
rows_1 = testing_common.AddRows(test_path1, [1476201840])
rows_2 = testing_common.AddRows(test_path2, [1476201870])
rows_3 = testing_common.AddRows(test_path2, [1476193390])
# These will be the revisions used to determine label.
rows_1[0].r_commit_pos = 112005
rows_2[0].r_commit_pos = 112010
rows_3[0].r_commit_pos = 112015
return (anomaly_key1, anomaly_key2, anomaly_key3)
def _AddAlertsToDataStore(self):
"""Adds sample data, including triaged and non-triaged alerts."""
key_map = {}
subscription = Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
testing_common.AddTests(
['ChromiumGPU'], ['linux-release'], {
'scrolling-benchmark': {
'first_paint': {},
'first_paint_ref': {},
'mean_frame_time': {},
'mean_frame_time_ref': {},
}
})
first_paint = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/first_paint')
mean_frame_time = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time')
# By default, all TestMetadata entities have an improvement_direction of
# UNKNOWN, meaning that neither direction is considered an improvement.
# Here we set the improvement direction so that some anomalies are
# considered improvements.
for test_key in [first_paint, mean_frame_time]:
test = test_key.get()
test.improvement_direction = anomaly.DOWN
test.put()
# Add some (12) non-triaged alerts.
for end_rev in range(10000, 10120, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
anomaly_entity = anomaly.Anomaly(
start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
ref_test=ref_test_key,
subscriptions=[subscription],
subscription_names=[subscription.name],
)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
# Add some (2) already-triaged alerts.
for end_rev in range(10120, 10140, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
bug_id = -1 if end_rev % 20 == 0 else 12345
anomaly_entity = anomaly.Anomaly(
start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
ref_test=ref_test_key,
bug_id=bug_id,
subscriptions=[subscription],
subscription_names=[subscription.name],
)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
if bug_id > 0:
bug_data.Bug.New(project='chromium', bug_id=bug_id).put()
# Add some (6) non-triaged improvements.
for end_rev in range(10140, 10200, 10):
test_key = mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
anomaly_entity = anomaly.Anomaly(
start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=200,
median_after_anomaly=100,
ref_test=ref_test_key,
subscriptions=[subscription],
subscription_names=[subscription.name],
)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
self.assertTrue(anomaly_entity.is_improvement)
key_map[end_rev] = anomaly_key.urlsafe()
return key_map
def testProcessTest(self, mock_email_sheriff):
self._AddDataForTests()
test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref'
test = utils.TestKey(test_path).get()
test.UpdateSheriff()
test.put()
alert_group_key1 = alert_group.AlertGroup(
name='scrolling_benchmark',
subscription_name='sheriff1',
status=alert_group.AlertGroup.Status.untriaged,
active=True,
revision=alert_group.RevisionRange(repository='chromium',
start=10000,
end=10070),
).put()
alert_group_key2 = alert_group.AlertGroup(
name='scrolling_benchmark',
subscription_name='sheriff2',
status=alert_group.AlertGroup.Status.untriaged,
active=True,
revision=alert_group.RevisionRange(repository='chromium',
start=10000,
end=10070),
).put()
s1 = Subscription(name='sheriff1', visibility=VISIBILITY.PUBLIC)
s2 = Subscription(name='sheriff2', visibility=VISIBILITY.PUBLIC)
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([s1, s2],
None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.key.id())])
self.ExecuteDeferredTasks('default')
expected_calls = [
mock.call([ModelMatcher('sheriff1'),
ModelMatcher('sheriff2')],
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10011)),
mock.call([ModelMatcher('sheriff1'),
ModelMatcher('sheriff2')],
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10041)),
mock.call([ModelMatcher('sheriff1'),
ModelMatcher('sheriff2')],
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10061))
]
self.assertEqual(expected_calls, mock_email_sheriff.call_args_list)
anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(len(anomalies), 3)
for a in anomalies:
self.assertEqual(a.groups, [alert_group_key1, alert_group_key2])
def AnomalyExists(anomalies, test, percent_changed, direction,
start_revision, end_revision, subscription_names,
internal_only, units, absolute_delta, statistic):
for a in anomalies:
if (a.test == test and a.percent_changed == percent_changed
and a.direction == direction
and a.start_revision == start_revision
and a.end_revision == end_revision
and a.subscription_names == subscription_names
and a.internal_only == internal_only
and a.units == units
and a.absolute_delta == absolute_delta
and a.statistic == statistic):
return True
return False
self.assertTrue(
AnomalyExists(anomalies,
test.key,
percent_changed=100,
direction=anomaly.UP,
start_revision=10007,
end_revision=10011,
subscription_names=['sheriff1', 'sheriff2'],
internal_only=False,
units='ms',
absolute_delta=50,
statistic='avg'))
self.assertTrue(
AnomalyExists(anomalies,
test.key,
percent_changed=-50,
direction=anomaly.DOWN,
start_revision=10037,
end_revision=10041,
subscription_names=['sheriff1', 'sheriff2'],
internal_only=False,
units='ms',
absolute_delta=-100,
statistic='avg'))
self.assertTrue(
AnomalyExists(anomalies,
test.key,
percent_changed=sys.float_info.max,
direction=anomaly.UP,
start_revision=10057,
end_revision=10061,
internal_only=False,
units='ms',
subscription_names=['sheriff1', 'sheriff2'],
absolute_delta=100,
statistic='avg'))
# This is here just to verify that AnomalyExists returns False sometimes.
self.assertFalse(
AnomalyExists(anomalies,
test.key,
percent_changed=100,
direction=anomaly.DOWN,
start_revision=10037,
end_revision=10041,
subscription_names=['sheriff1', 'sheriff2'],
internal_only=False,
units='ms',
absolute_delta=500,
statistic='avg'))
def _Subscription(self, suffix=""):
"""Adds a Sheriff entity and returns the key."""
return Subscription(name='Chromium Perf Sheriff' + suffix,
notification_email='*****@*****.**')
def testGet_UsesFirstDefinedComponent(self):
ownership_samples = [{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': ''
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': 'Abc>Def'
}]
now_datetime = datetime.datetime.now()
test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint')
subscription = Subscription(
name='Sheriff',
bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript'])
alert_without_ownership = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
timestamp=now_datetime).put()
alert_without_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[0],
timestamp=now_datetime + datetime.timedelta(10)).put()
alert_with_empty_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[1],
timestamp=now_datetime + datetime.timedelta(20)).put()
alert_with_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
subscriptions=[subscription],
subscription_names=[subscription.name],
ownership=ownership_samples[2],
timestamp=now_datetime + datetime.timedelta(30)).put()
response = self.testapp.post('/file_bug', [
('keys', '%s,%s,%s,%s' % (alert_without_ownership.urlsafe(),
alert_without_component.urlsafe(),
alert_with_empty_component.urlsafe(),
alert_with_component.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response.body)