def testPost_ExternalUserRequestsInternalOnlySheriff_ErrorMessage(self):
sheriff.Sheriff(id='Foo', internal_only=True).put()
self.assertFalse(utils.IsInternalUser())
response = self.testapp.post('/alerts?sheriff=Foo')
error = self.GetJsonValue(response, 'error')
self.assertIsNotNone(error)
def testProcessTest(self, mock_email_sheriff):
self._AddDataForTests()
test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref'
test = utils.TestKey(test_path).get()
sheriff.Sheriff(
email='*****@*****.**', id='sheriff', patterns=[test_path]).put()
test.UpdateSheriff()
test.put()
find_anomalies.ProcessTests([test.key])
self.ExecuteDeferredTasks('default')
expected_calls = [
mock.call(ModelMatcher('sheriff'),
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10011)),
mock.call(ModelMatcher('sheriff'),
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10041)),
mock.call(ModelMatcher('sheriff'),
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10061))]
self.assertEqual(expected_calls, mock_email_sheriff.call_args_list)
anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(len(anomalies), 3)
def AnomalyExists(
anomalies, test, percent_changed, direction,
start_revision, end_revision, sheriff_name, internal_only, units,
absolute_delta, statistic):
for a in anomalies:
if (a.test == test and
a.percent_changed == percent_changed and
a.direction == direction and
a.start_revision == start_revision and
a.end_revision == end_revision and
a.sheriff.string_id() == sheriff_name and
a.internal_only == internal_only and
a.units == units and
a.absolute_delta == absolute_delta and
a.statistic == statistic):
return True
return False
self.assertTrue(
AnomalyExists(
anomalies, test.key, percent_changed=100, direction=anomaly.UP,
start_revision=10007, end_revision=10011, sheriff_name='sheriff',
internal_only=False, units='ms', absolute_delta=50,
statistic='avg'))
self.assertTrue(
AnomalyExists(
anomalies, test.key, percent_changed=-50, direction=anomaly.DOWN,
start_revision=10037, end_revision=10041, sheriff_name='sheriff',
internal_only=False, units='ms', absolute_delta=-100,
statistic='avg'))
self.assertTrue(
AnomalyExists(
anomalies, test.key, percent_changed=sys.float_info.max,
direction=anomaly.UP, start_revision=10057, end_revision=10061,
sheriff_name='sheriff', internal_only=False, units='ms',
absolute_delta=100, statistic='avg'))
# This is here just to verify that AnomalyExists returns False sometimes.
self.assertFalse(
AnomalyExists(
anomalies, test.key, percent_changed=100, direction=anomaly.DOWN,
start_revision=10037, end_revision=10041, sheriff_name='sheriff',
internal_only=False, units='ms', absolute_delta=500,
statistic='avg'))
def _AddSheriffs(self):
sheriff1 = sheriff.Sheriff(id='Chromium Perf Sheriff',
email='*****@*****.**').put()
sheriff2 = sheriff.Sheriff(id='QA Perf Sheriff',
email='*****@*****.**').put()
return [sheriff1, sheriff2]
def testGet_WithNoAlerts_HasImageAndNoAlertsTable(self):
sheriff.Sheriff(id='Chromium Perf Sheriff').put()
response = self.testapp.get('/alerts')
self.assertEqual(1, len(response.html('img')))
self.assertEqual(0, len(response.html('alerts-table')))
def testSheriffEmail_InternalOnly_OtherDomain_RaisesError(self):
# If it's not within the set of allowed domains, an exception will be raised
# while putting.
entity = sheriff.Sheriff(internal_only=True, email='*****@*****.**')
with self.assertRaises(sheriff.ValidationError):
entity.put()
def _AddAlertsToDataStore(self):
"""Adds sample data, including triaged and non-triaged alerts."""
key_map = {}
subscription = Subscription(
name='Chromium Perf Sheriff',
notification_email='*****@*****.**',
)
# We still need sheriff information from here before sheriff-config
# provide a way to fetch subsciber list.
sheriff.Sheriff(
id='Chromium Perf Sheriff', email='*****@*****.**').put()
testing_common.AddTests(['ChromiumGPU'], ['linux-release'], {
'scrolling-benchmark': {
'first_paint': {},
'first_paint_ref': {},
'mean_frame_time': {},
'mean_frame_time_ref': {},
}
})
first_paint = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/first_paint')
mean_frame_time = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time')
# By default, all TestMetadata entities have an improvement_direction of
# UNKNOWN, meaning that neither direction is considered an improvement.
# Here we set the improvement direction so that some anomalies are
# considered improvements.
for test_key in [first_paint, mean_frame_time]:
test = test_key.get()
test.improvement_direction = anomaly.DOWN
test.put()
# Add some (12) non-triaged alerts.
for end_rev in range(10000, 10120, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
anomaly_entity = anomaly.Anomaly(
start_revision=end_rev - 5, end_revision=end_rev, test=test_key,
median_before_anomaly=100, median_after_anomaly=200,
ref_test=ref_test_key,
subscriptions=[subscription],
subscription_names=[subscription.name],
)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
# Add some (2) already-triaged alerts.
for end_rev in range(10120, 10140, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
bug_id = -1 if end_rev % 20 == 0 else 12345
anomaly_entity = anomaly.Anomaly(
start_revision=end_rev - 5, end_revision=end_rev, test=test_key,
median_before_anomaly=100, median_after_anomaly=200,
ref_test=ref_test_key, bug_id=bug_id,
subscriptions=[subscription],
subscription_names=[subscription.name],
)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
if bug_id > 0:
bug_data.Bug(id=bug_id).put()
# Add some (6) non-triaged improvements.
for end_rev in range(10140, 10200, 10):
test_key = mean_frame_time
ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key))
anomaly_entity = anomaly.Anomaly(
start_revision=end_rev - 5, end_revision=end_rev, test=test_key,
median_before_anomaly=200, median_after_anomaly=100,
ref_test=ref_test_key,
subscriptions=[subscription],
subscription_names=[subscription.name],
)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
self.assertTrue(anomaly_entity.is_improvement)
key_map[end_rev] = anomaly_key.urlsafe()
return key_map
def testGet_WithAllOwnershipComponents(self):
ownership_samples = [
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': 'Abc>Xyz'
},
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'component': 'Def>123'
}
]
test_paths = ['ChromiumPerf/linux/scrolling/first_paint',
'ChromiumPerf/linux/scrolling/mean_frame_time']
test_keys = [utils.TestKey(test_path) for test_path in test_paths]
sheriff_key = sheriff.Sheriff(
id='Sheriff',
labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put()
anomaly_1 = anomaly.Anomaly(
start_revision=1476193324, end_revision=1476201840, test=test_keys[0],
median_before_anomaly=100, median_after_anomaly=200,
sheriff=sheriff_key, ownership=ownership_samples[0]).put()
anomaly_2 = anomaly.Anomaly(
start_revision=1476193320, end_revision=1476201870, test=test_keys[1],
median_before_anomaly=100, median_after_anomaly=200,
sheriff=sheriff_key, ownership=ownership_samples[1]).put()
response = self.testapp.post(
'/file_bug',
[
('keys', '%s' % (anomaly_1.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Xyz">',
response.body)
response_with_both_anomalies = self.testapp.post(
'/file_bug',
[
('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Xyz">',
response_with_both_anomalies.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="Def>123">',
response_with_both_anomalies.body)
def testProcessTest(self, mock_email_sheriff):
self._AddDataForTests()
test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref'
test = utils.TestKey(test_path).get()
sheriff.Sheriff(email='*****@*****.**',
id='sheriff',
patterns=[test_path]).put()
test.UpdateSheriff()
test.put()
s1 = Subscription(name='sheriff1', visibility=VISIBILITY.PUBLIC)
s2 = Subscription(name='sheriff2', visibility=VISIBILITY.PUBLIC)
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([s1, s2],
None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.key.id())])
self.ExecuteDeferredTasks('default')
expected_calls = [
mock.call([ModelMatcher('sheriff1'),
ModelMatcher('sheriff2')],
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10011)),
mock.call([ModelMatcher('sheriff1'),
ModelMatcher('sheriff2')],
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10041)),
mock.call([ModelMatcher('sheriff1'),
ModelMatcher('sheriff2')],
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10061))
]
self.assertEqual(expected_calls, mock_email_sheriff.call_args_list)
anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(len(anomalies), 3)
def AnomalyExists(anomalies, test, percent_changed, direction,
start_revision, end_revision, subscription_names,
internal_only, units, absolute_delta, statistic):
for a in anomalies:
if (a.test == test and a.percent_changed == percent_changed
and a.direction == direction
and a.start_revision == start_revision
and a.end_revision == end_revision
and a.subscription_names == subscription_names
and a.internal_only == internal_only
and a.units == units
and a.absolute_delta == absolute_delta
and a.statistic == statistic):
return True
return False
self.assertTrue(
AnomalyExists(anomalies,
test.key,
percent_changed=100,
direction=anomaly.UP,
start_revision=10007,
end_revision=10011,
subscription_names=['sheriff1', 'sheriff2'],
internal_only=False,
units='ms',
absolute_delta=50,
statistic='avg'))
self.assertTrue(
AnomalyExists(anomalies,
test.key,
percent_changed=-50,
direction=anomaly.DOWN,
start_revision=10037,
end_revision=10041,
subscription_names=['sheriff1', 'sheriff2'],
internal_only=False,
units='ms',
absolute_delta=-100,
statistic='avg'))
self.assertTrue(
AnomalyExists(anomalies,
test.key,
percent_changed=sys.float_info.max,
direction=anomaly.UP,
start_revision=10057,
end_revision=10061,
internal_only=False,
units='ms',
subscription_names=['sheriff1', 'sheriff2'],
absolute_delta=100,
statistic='avg'))
# This is here just to verify that AnomalyExists returns False sometimes.
self.assertFalse(
AnomalyExists(anomalies,
test.key,
percent_changed=100,
direction=anomaly.DOWN,
start_revision=10037,
end_revision=10041,
subscription_names=['sheriff1', 'sheriff2'],
internal_only=False,
units='ms',
absolute_delta=500,
statistic='avg'))
def testProcessTest_RefineAnomalyPlacement_OffByOneBefore(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'],
{'blink_perf.layout': {
'nested-percent-height-tables': {}
}})
test = utils.TestKey(
'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables'
).get()
test_container_key = utils.GetTestContainerKey(test.key)
sample_data = [
(728446, 480.2504),
(728462, 487.685),
(728469, 486.6389),
(728480, 477.6597),
(728492, 471.2238),
(728512, 480.4379),
(728539, 464.5573),
(728594, 489.0594),
(728644, 484.4796),
(728714, 489.5986),
(728751, 489.474),
(728788, 481.9336),
(728835, 484.089),
(728869, 485.4287),
(728883, 476.8234),
(728907, 487.4736),
(728938, 490.601),
(728986, 483.5039),
(729021, 485.176),
(729066, 484.5855),
(729105, 483.9114),
(729119, 483.559),
(729161, 477.6875),
(729201, 484.9668),
(729240, 480.7091),
(729270, 484.5506),
(729292, 495.1445),
(729309, 479.9111),
(729329, 479.8815),
(729391, 487.5683),
(729430, 476.7355),
(729478, 487.7251),
(729525, 493.1012),
(729568, 497.7565),
(729608, 499.6481),
(729642, 496.1591),
(729658, 493.4581),
(729687, 486.1097),
(729706, 478.036),
(729730,
480.4222), # In crbug/1041688 this was the original placement.
(729764, 421.0342), # We instead should be setting it here.
(729795, 428.0284),
(729846, 433.8261),
(729883, 429.49),
(729920, 436.3342),
(729975, 434.3996),
(730011, 428.3672),
(730054, 436.309),
(730094, 435.3792),
(730128, 433.0537),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
sheriff.Sheriff(email='*****@*****.**',
id='sheriff',
patterns=[test.test_path]).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.DOWN, new_anomalies[0].direction)
self.assertEqual(729731, new_anomalies[0].start_revision)
self.assertEqual(729764, new_anomalies[0].end_revision)
def testGet_UsesFirstDefinedComponent(self):
ownership_samples = [{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': ''
}, {
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': 'Abc>Def'
}]
now_datetime = datetime.datetime.now()
test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint')
sheriff_key = sheriff.Sheriff(
id='Sheriff',
labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put()
alert_without_ownership = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
timestamp=now_datetime).put()
alert_without_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
ownership=ownership_samples[0],
timestamp=now_datetime + datetime.timedelta(10)).put()
alert_with_empty_component = anomaly.Anomaly(
start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
ownership=ownership_samples[1],
timestamp=now_datetime + datetime.timedelta(20)).put()
alert_with_component = anomaly.Anomaly(start_revision=1476193320,
end_revision=1476201870,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key,
ownership=ownership_samples[2],
timestamp=now_datetime +
datetime.timedelta(30)).put()
response = self.testapp.post('/file_bug', [
('keys', '%s,%s,%s,%s' % (alert_without_ownership.urlsafe(),
alert_without_component.urlsafe(),
alert_with_empty_component.urlsafe(),
alert_with_component.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response.body)
def setUp(self):
super(BenchFindChangePointsTest, self).setUp()
self.sheriff = sheriff.Sheriff(
email='*****@*****.**', id=bench_find_anomalies._TEST_DATA_SHERIFF).put()
def _AddSheriffToDatastore(self):
sheriff_key = sheriff.Sheriff(id='Chromium Perf Sheriff',
email='*****@*****.**')
sheriff_key.patterns = ['*/*/my_test_suite/*']
sheriff_key.put()
def _AddDataToDatastore(self):
"""Puts a set of entities; some internal-only, some not."""
# Need to be privileged to add TestMetadata and Row objects to the datastore
# because there is a get() for the parent_test in the pre_put_hook. This
# should work correctly in production because Rows and TestMetadata should
# only be added by /add_point, which is privileged.
self.SetCurrentUser('*****@*****.**')
testing_common.AddTests(
['ChromiumPerf'], ['Win7External', 'FooInternal'], {
'TestInternal': {
'SubTestInternal': {}
},
'TestExternal': {
'SubTestExternal': {}
},
})
internal_key = ['Master', 'ChromiumPerf', 'Bot', 'FooInternal']
internal_bot = ndb.Key(*internal_key).get()
internal_bot.internal_only = True
internal_bot.put()
internal_test = ndb.Key(
'TestMetadata', 'ChromiumPerf/Win7External/TestInternal').get()
internal_test.internal_only = True
internal_test.put()
internal_test = ndb.Key('TestMetadata',
'ChromiumPerf/FooInternal/TestInternal').get()
internal_test.internal_only = True
internal_test.put()
internal_sub_test = ndb.Key(
'TestMetadata',
'ChromiumPerf/Win7External/TestInternal/SubTestInternal').get()
internal_sub_test.internal_only = True
internal_sub_test.put()
internal_sub_test = ndb.Key(
'TestMetadata',
'ChromiumPerf/FooInternal/TestInternal/SubTestInternal').get()
internal_sub_test.internal_only = True
internal_sub_test.put()
internal_key = internal_sub_test.key
external_key = ndb.Key(
'TestMetadata',
'ChromiumPerf/Win7External/TestExternal/SubTestExternal')
internal_test_container_key = utils.GetTestContainerKey(internal_key)
external_test_container_key = utils.GetTestContainerKey(external_key)
for i in range(0, 100, 10):
graph_data.Row(parent=internal_test_container_key,
id=i,
value=float(i * 2),
internal_only=True).put()
graph_data.Row(parent=external_test_container_key,
id=i,
value=float(i * 2)).put()
self.UnsetCurrentUser()
sheriff.Sheriff(id='external',
email='*****@*****.**',
internal_only=False).put()
sheriff.Sheriff(id='internal',
email='*****@*****.**',
internal_only=True).put()
def _AddMockAlertSummaryData(self):
"""Adds data to be used in the alert-summary stats tests below."""
correct_sheriff = sheriff.Sheriff(id='Chromium Perf Sheriff',
email='*****@*****.**',
patterns=[]).put()
wrong_sheriff = sheriff.Sheriff(id='Some other sheriff',
email='*****@*****.**',
patterns=[]).put()
linux_sunspider = 'ChromiumPerf/linux-release/sunspider/Total'
linux_octane = 'ChromiumPerf/linux-release/octane/Total'
linux_media = 'ChromiumPerf/linux-release/media.tough_media_cases/Total'
windows_sunspider = 'ChromiumPerf/windows/sunspider/Total'
windows_octane = 'ChromiumPerf/windows/octane/Total'
# Should not be included: too early.
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 1),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
# Should not be included: too late.
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 6, 17),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
# Should not be included: wrong sheriff.
anomaly.Anomaly(sheriff=wrong_sheriff,
timestamp=datetime.datetime(2013, 5, 11),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
# Everything below should be included.
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=100.5).put()
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(linux_octane),
median_before_anomaly=100,
median_after_anomaly=101.5,
bug_id=-1).put()
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(linux_media),
median_before_anomaly=100,
median_after_anomaly=101.5,
bug_id=-2).put()
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(windows_sunspider),
median_before_anomaly=100,
median_after_anomaly=104.5,
bug_id=12345).put()
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 12),
test=utils.TestKey(windows_octane),
median_before_anomaly=100,
median_after_anomaly=600,
bug_id=12345).put()
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 15),
test=utils.TestKey(linux_sunspider),
median_before_anomaly=100,
median_after_anomaly=200).put()
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 20),
test=utils.TestKey(windows_sunspider),
median_before_anomaly=100,
median_after_anomaly=115,
bug_id=12345).put()
anomaly.Anomaly(sheriff=correct_sheriff,
timestamp=datetime.datetime(2013, 5, 21),
test=utils.TestKey(windows_octane),
median_before_anomaly=100,
median_after_anomaly=104).put()
def _AddAlertsToDataStore(self):
"""Adds sample data, including triaged and non-triaged alerts."""
key_map = {}
sheriff_key = sheriff.Sheriff(id='Chromium Perf Sheriff',
email='*****@*****.**').put()
testing_common.AddTests(['ChromiumGPU'], ['linux-release'], {
'scrolling-benchmark': {
'first_paint': {},
'mean_frame_time': {},
}
})
first_paint = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/first_paint')
mean_frame_time = utils.TestKey(
'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time')
# By default, all TestMetadata entities have an improvement_direction of
# UNKNOWN, meaning that neither direction is considered an improvement.
# Here we set the improvement direction so that some anomalies are
# considered improvements.
for test_key in [first_paint, mean_frame_time]:
test = test_key.get()
test.improvement_direction = anomaly.DOWN
test.put()
# Add some (12) non-triaged alerts.
for end_rev in range(10000, 10120, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
sheriff=sheriff_key)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
# Add some (2) already-triaged alerts.
for end_rev in range(10120, 10140, 10):
test_key = first_paint if end_rev % 20 == 0 else mean_frame_time
bug_id = -1 if end_rev % 20 == 0 else 12345
anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200,
bug_id=bug_id,
sheriff=sheriff_key)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
key_map[end_rev] = anomaly_key.urlsafe()
if bug_id > 0:
bug_data.Bug(id=bug_id).put()
# Add some (6) non-triaged improvements.
for end_rev in range(10140, 10200, 10):
test_key = mean_frame_time
anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5,
end_revision=end_rev,
test=test_key,
median_before_anomaly=200,
median_after_anomaly=100,
sheriff=sheriff_key)
anomaly_entity.SetIsImprovement()
anomaly_key = anomaly_entity.put()
self.assertTrue(anomaly_entity.is_improvement)
key_map[end_rev] = anomaly_key.urlsafe()
return key_map
def testProcessTest_RefineAnomalyPlacement_OffByOneStable(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'], {
'memory.desktop': {
('memory:chrome:all_processes:'
'reported_by_chrome:v8:effective_size_avg'): {}
}
})
test = utils.TestKey((
'ChromiumPerf/linux-perf/memory.desktop/'
'memory:chrome:all_processes:reported_by_chrome:v8:effective_size_avg'
)).get()
test_container_key = utils.GetTestContainerKey(test.key)
sample_data = [
(733480, 1381203.0),
(733494, 1381220.0),
(733504, 1381212.0),
(733524, 1381220.0),
(733538, 1381211.0),
(733544, 1381212.0),
(733549, 1381220.0),
(733563, 1381220.0),
(733581, 1381220.0),
(733597, 1381212.0),
(733611, 1381228.0),
(733641, 1381212.0),
(733675, 1381204.0),
(733721, 1381212.0),
(733766, 1381211.0),
(733804, 1381204.0),
(733835, 1381219.0),
(733865, 1381211.0),
(733885, 1381219.0),
(733908, 1381204.0),
(733920, 1381211.0),
(733937, 1381220.0),
(734091, 1381211.0),
(734133, 1381219.0),
(734181, 1381204.0),
(734211, 1381720.0),
(734248, 1381712.0),
(734277, 1381696.0),
(734311, 1381704.0),
(734341, 1381703.0),
(734372, 1381704.0),
(734405, 1381703.0),
(734431, 1381711.0),
(734456, 1381720.0),
(734487, 1381703.0),
(734521, 1381704.0),
(734554, 1381726.0),
(734598, 1381704.0),
(734630,
1381703.0), # In crbug/1041688 this is where it was placed.
(734673, 1529888.0), # This is where it should be.
(734705, 1529888.0),
(734739, 1529860.0),
(734770, 1529860.0),
(734793, 1529888.0),
(734829, 1529860.0),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
sheriff.Sheriff(email='*****@*****.**',
id='sheriff',
patterns=[test.test_path]).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.UP, new_anomalies[0].direction)
self.assertEqual(734631, new_anomalies[0].start_revision)
self.assertEqual(734673, new_anomalies[0].end_revision)
def _AddSheriff(self):
"""Adds a Sheriff entity and returns the key."""
return sheriff.Sheriff(id='Chromium Perf Sheriff',
email='*****@*****.**').put()
def testSheriffUrlValidator_ValidUrl_DoesntRaiseError(self):
# By contrast, this URL should be accepted.
sheriff.Sheriff(url='http://x.com/')
def testGet_OwnersAreEmptyEvenWithOwnership(self):
ownership_samples = [
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'emails': ['*****@*****.**']
},
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'emails': ['*****@*****.**']
}
]
test_paths = ['ChromiumPerf/linux/scrolling/first_paint',
'ChromiumPerf/linux/scrolling/mean_frame_time']
test_keys = [utils.TestKey(test_path) for test_path in test_paths]
sheriff_key = sheriff.Sheriff(
id='Sheriff',
labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put()
anomaly_1 = anomaly.Anomaly(
start_revision=1476193324, end_revision=1476201840, test=test_keys[0],
median_before_anomaly=100, median_after_anomaly=200,
sheriff=sheriff_key, ownership=ownership_samples[0]).put()
anomaly_2 = anomaly.Anomaly(
start_revision=1476193320, end_revision=1476201870, test=test_keys[1],
median_before_anomaly=100, median_after_anomaly=200,
sheriff=sheriff_key, ownership=ownership_samples[1]).put()
response = self.testapp.post(
'/file_bug',
[
('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="text" name="owner" value="">',
response.body)
response_changed_order = self.testapp.post(
'/file_bug',
[
('keys', '%s,%s' % (anomaly_2.urlsafe(), anomaly_1.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertIn(
'<input type="text" name="owner" value="">',
response_changed_order.body)
def testSheriffEmail_InvalidEmail_RaisesError(self):
# If the given email address is not valid, an assertion error
# will be raised while creating the entity
with self.assertRaises(sheriff.ValidationError):
sheriff.Sheriff(email='oops')
def testGet_UsesOnlyMostRecentComponents(self):
ownership_samples = [
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827',
'component': 'Abc>Def'
},
{
'type': 'Ownership',
'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826',
'component': '123>456'
},
]
sheriff_key = sheriff.Sheriff(
id='Sheriff',
labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put()
test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint')
now_datetime = datetime.datetime.now()
older_alert = anomaly.Anomaly(
start_revision=1476193320, end_revision=1476201870, test=test_key,
median_before_anomaly=100, median_after_anomaly=200,
sheriff=sheriff_key, ownership=ownership_samples[0],
timestamp=now_datetime).put()
newer_alert = anomaly.Anomaly(
start_revision=1476193320, end_revision=1476201870, test=test_key,
median_before_anomaly=100, median_after_anomaly=200,
sheriff=sheriff_key, ownership=ownership_samples[1],
timestamp=now_datetime + datetime.timedelta(10)).put()
response = self.testapp.post(
'/file_bug',
[
('keys', '%s,%s' % (older_alert.urlsafe(),
newer_alert.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertNotIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="123>456">',
response.body)
response_inverted_order = self.testapp.post(
'/file_bug',
[
('keys', '%s,%s' % (newer_alert.urlsafe(),
older_alert.urlsafe())),
('summary', 's'),
('description', 'd\n'),
('label', 'one'),
('label', 'two'),
('component', 'Foo>Bar'),
])
self.assertNotIn(
'<input type="checkbox" checked name="component" value="Abc>Def">',
response_inverted_order.body)
self.assertIn(
'<input type="checkbox" checked name="component" value="123>456">',
response_inverted_order.body)
def _AddSheriff(self, name, email=None, url=None,
internal_only=False, summarize=False, patterns=None):
"""Adds a Sheriff entity to the datastore."""
sheriff.Sheriff(
id=name, email=email, url=url, internal_only=internal_only,
summarize=summarize, patterns=patterns or []).put()
