def _AddAnomalyConfig(self, config_name, test_key, config_dict):
"""Adds a custom anomaly config which applies to one test."""
anomaly_config_key = anomaly_config.AnomalyConfig(
id=config_name,
config=config_dict,
patterns=[utils.TestPath(test_key)]).put()
return anomaly_config_key
def testPinpointParams_OverriddenAnomalyConfig(self):
params = {
'test_path':
'ChromiumPerf/Android Nexus5X WebView Perf/system_health/foo',
'start_commit': '1051',
'end_commit': '1151',
'bug_id': 1,
'bisect_mode': 'performance',
'story_filter': '',
'pin': '',
}
a = anomaly_config.AnomalyConfig()
a.config = {'min_segment_size': 1}
a.patterns = ['*/*/*/*']
a.put()
t = graph_data.TestMetadata(id=params['test_path'])
t.overridden_anomaly_config = a.key
t.put()
rows = dict(
itertools.chain(
zip(itertools.islice(itertools.count(1000, 2), 75),
itertools.repeat({'value': -100.0})), [(1050, {
'value': 0.1
})],
zip(itertools.islice(itertools.count(1101, 2), 50),
itertools.repeat({'value': 0.5}))))
testing_common.AddRows(params['test_path'], rows)
results = pinpoint_request.PinpointParamsFromBisectParams(params)
# We overrode the anomaly config with a window of 1, and there's only a
# single row with value 0.1, the rest are 0.0.
self.assertEqual(0.4, results['comparison_magnitude'])
def testGetAnomalyConfigDict(self):
testing_common.AddTests(['M'], ['b'], {'foo': {'bar': {}}})
test = utils.TestKey('M/b/foo/bar').get()
# The sample test has no overridden config.
self.assertEqual({}, anomaly_config.GetAnomalyConfigDict(test))
# Override the config for the test added above.
# The overridden config is set in the pre-put hook of the Test.
my_config = {
'_comment': 'Very particular segment sizes.',
'max_window_size': 721,
'min_segment_size': 123,
}
my_patterns = [test.test_path]
anomaly_config.AnomalyConfig(config=my_config,
patterns=my_patterns).put()
test.put()
# The sample test now has an overridden config which is used.
# Extraneous "comment" keys are ignored.
expected = {
'max_window_size': 721,
'min_segment_size': 123,
}
self.assertEqual(expected, anomaly_config.GetAnomalyConfigDict(test))
def testProcessTest_RefineAnomalyPlacement_MinSize0Max2Elements(self):
testing_common.AddTests(['ChromiumPerf'], ['linux-perf'],
{'sizes': {
'method_count': {}
}})
test = utils.TestKey(
('ChromiumPerf/linux-perf/sizes/method_count')).get()
test_container_key = utils.GetTestContainerKey(test.key)
custom_config = {
'max_window_size': 10,
'min_absolute_change': 50,
'min_relative_change': 0,
'min_segment_size': 0,
}
anomaly_config.AnomalyConfig(config=custom_config,
patterns=[test.test_path]).put()
test.UpdateSheriff()
test.put()
self.assertEqual(custom_config,
anomaly_config.GetAnomalyConfigDict(test))
sample_data = [
(6990, 100),
(6991, 100),
(6992, 100),
(6993, 100),
(6994, 100),
(6995, 100),
(6996, 100),
(6997, 100),
(6998, 100),
(6999, 100),
(7000, 100),
(7001, 155),
(7002, 155),
(7003, 155),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
sheriff.Sheriff(email='*****@*****.**',
id='sheriff',
patterns=[test.test_path]).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.UP, new_anomalies[0].direction)
self.assertEqual(7001, new_anomalies[0].start_revision)
self.assertEqual(7001, new_anomalies[0].end_revision)
def testEdit_RemovePattern(self):
"""Tests removing a pattern from an AnomalyConfig."""
self.SetCurrentUser('*****@*****.**', is_admin=True)
anomaly_config_key = anomaly_config.AnomalyConfig(
id='Test Config',
config={
'a': 10
},
patterns=['*/*/one', '*/*/two']).put()
master = graph_data.Master(id='TheMaster').put()
graph_data.Bot(id='TheBot', parent=master).put()
test_one = graph_data.TestMetadata(
id='TheMaster/TheBot/one',
overridden_anomaly_config=anomaly_config_key,
has_rows=True)
test_one.UpdateSheriff()
test_one = test_one.put()
test_two = graph_data.TestMetadata(
id='TheMaster/TheBot/two',
overridden_anomaly_config=anomaly_config_key,
has_rows=True)
test_two.UpdateSheriff()
test_two = test_two.put()
# Verify the state of the data before making the request.
self.assertEqual(['*/*/one', '*/*/two'],
anomaly_config_key.get().patterns)
self.assertEqual(['TheMaster/TheBot/one'],
list_tests.GetTestsMatchingPattern('*/*/one'))
self.testapp.post(
'/edit_anomaly_configs', {
'add-edit': 'edit',
'edit-name': 'Test Config',
'config': '{"a": 10}',
'patterns': ['*/*/two'],
'xsrf_token': xsrf.GenerateToken(users.get_current_user()),
})
self.ExecuteDeferredTasks('default')
self.ExecuteTaskQueueTasks('/put_entities_task',
edit_config_handler._TASK_QUEUE_NAME)
self.assertEqual(['*/*/two'], anomaly_config_key.get().patterns)
self.assertIsNone(test_one.get().overridden_anomaly_config)
self.assertEqual('Test Config',
test_two.get().overridden_anomaly_config.string_id())
def testPost_UpdatesAnomalyConfig(self):
testing_common.AddTests(*_TESTS)
anomaly_config.AnomalyConfig(
id='anomaly_config1',
config='',
patterns=['ChromiumPerf/mac/SunSpider/Total']).put()
t = utils.TestKey('ChromiumPerf/mac/SunSpider/Total').get()
self.assertIsNone(t.sheriff)
self.assertIsNone(t.overridden_anomaly_config)
self.testapp.post('/cron/update_sheriff')
self.ExecuteDeferredTasks(cron_update_sheriff._TASK_QUEUE_NAME)
t = utils.TestKey('ChromiumPerf/mac/SunSpider/Total').get()
self.assertIsNone(t.sheriff)
self.assertIsNotNone(t.overridden_anomaly_config)
def testEdit(self):
"""Tests changing the config property of an existing AnomalyConfig."""
self.SetCurrentUser('*****@*****.**', is_admin=True)
anomaly_config.AnomalyConfig(
id='Existing Config', config={'old': 11},
patterns=['MyMaster/*/*/*']).put()
self.testapp.post('/edit_anomaly_configs', {
'add-edit': 'edit',
'edit-name': 'Existing Config',
'config': '{"new": 10}',
'patterns': 'MyMaster/*/*/*',
'xsrf_token': xsrf.GenerateToken(users.get_current_user()),
})
anomaly_configs = anomaly_config.AnomalyConfig.query().fetch()
self.assertEqual(len(anomaly_configs), 1)
self.assertEqual('Existing Config', anomaly_configs[0].key.string_id())
self.assertEqual({'new': 10}, anomaly_configs[0].config)
self.assertEqual(['MyMaster/*/*/*'], anomaly_configs[0].patterns)
def _AddTestData(self,
test_name,
rows,
improvement_direction=anomaly.UNKNOWN,
config=None):
testing_common.AddTests(['ChromiumGPU'], ['linux-release'], {
'scrolling_benchmark': {
test_name: {},
},
})
test = utils.TestKey('ChromiumGPU/linux-release/scrolling_benchmark/' +
test_name).get()
test.improvement_direction = improvement_direction
test_container_key = utils.GetTestContainerKey(test.key)
sheriff_entity = self.sheriff.get()
if sheriff_entity.patterns:
sheriff_entity.patterns.append(test.test_path)
else:
sheriff_entity.patterns = [test.test_path]
sheriff_entity.put()
for row in rows:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
# Add test config.
if not config:
config = {
'max_window_size': 50,
'multiple_of_std_dev': 3.5,
'min_relative_change': 0.1,
'min_absolute_change': 1.0,
'min_segment_size': 3,
}
anomaly_config.AnomalyConfig(id='config_' + test_name,
config=config,
patterns=[test.test_path]).put()
test.put()
return test
def _AddTestData(self,
test_name,
rows,
sheriff_key_key,
improvement_direction=anomaly.UNKNOWN):
"""Adds a sample Test and associated data and returns the Test."""
testing_common.AddTests(['ChromiumGPU'], ['linux-release'], {
'scrolling_benchmark': {
test_name: {},
},
})
test = utils.TestKey('ChromiumGPU/linux-release/scrolling_benchmark/' +
test_name).get()
test.improvement_direction = improvement_direction
test_container_key = utils.GetTestContainerKey(test.key)
sheriff_key = sheriff_key_key.get()
if sheriff_key.patterns:
sheriff_key.patterns.append(test.test_path)
else:
sheriff_key.patterns = [test.test_path]
sheriff_key.put()
for row in rows:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
# Add test config.
overridden_config = {
'min_relative_change': 0.1,
'min_absolute_change': 10.0
}
anomaly_config.AnomalyConfig(id='config_' + test_name,
config=overridden_config,
patterns=[test.test_path]).put()
test.put()
return test
def testEdit_AddPattern(self):
"""Tests changing the patterns list of an existing AnomalyConfig."""
self.SetCurrentUser('*****@*****.**', is_admin=True)
master = graph_data.Master(id='TheMaster').put()
graph_data.Bot(id='TheBot', parent=master).put()
suite1 = graph_data.TestMetadata(id='TheMaster/TheBot/Suite1')
suite1.UpdateSheriff()
suite1 = suite1.put()
suite2 = graph_data.TestMetadata(id='TheMaster/TheBot/Suite2')
suite2.UpdateSheriff()
suite2 = suite2.put()
test_aaa = graph_data.TestMetadata(
id='TheMaster/TheBot/Suite1/aaa', has_rows=True)
test_aaa.UpdateSheriff()
test_aaa = test_aaa.put()
test_bbb = graph_data.TestMetadata(
id='TheMaster/TheBot/Suite1/bbb', has_rows=True)
test_bbb.UpdateSheriff()
test_bbb = test_bbb.put()
test_ccc = graph_data.TestMetadata(
id='TheMaster/TheBot/Suite1/ccc', has_rows=True)
test_ccc.UpdateSheriff()
test_ccc = test_ccc.put()
test_ddd = graph_data.TestMetadata(
id='TheMaster/TheBot/Suite2/ddd', has_rows=True)
test_ddd.UpdateSheriff()
test_ddd = test_ddd.put()
anomaly_config.AnomalyConfig(id='1-Suite1-specific', config={'a': 10}).put()
anomaly_config.AnomalyConfig(id='2-Suite1-general', config={'b': 20}).put()
self.testapp.post('/edit_anomaly_configs', {
'add-edit': 'edit',
'edit-name': '1-Suite1-specific',
'config': '{"a": 10}',
'patterns': '*/*/Suite1/aaa',
'xsrf_token': xsrf.GenerateToken(users.get_current_user()),
})
self.ExecuteTaskQueueTasks(
'/put_entities_task', edit_config_handler._TASK_QUEUE_NAME)
self.testapp.post('/edit_anomaly_configs', {
'add-edit': 'edit',
'edit-name': '2-Suite1-general',
'config': '{"b": 20}',
'patterns': '*/*/Suite1/*',
'xsrf_token': xsrf.GenerateToken(users.get_current_user()),
})
self.ExecuteDeferredTasks('default')
self.ExecuteTaskQueueTasks(
'/put_entities_task', edit_config_handler._TASK_QUEUE_NAME)
# The lists of test patterns in the AnomalyConfig entities in the datastore
# should be set based on what was added in the two requests above.
self.assertEqual(
['*/*/Suite1/*'],
anomaly_config.AnomalyConfig.get_by_id('2-Suite1-general').patterns)
self.assertEqual(
['*/*/Suite1/aaa'],
anomaly_config.AnomalyConfig.get_by_id('1-Suite1-specific').patterns)
# The 1-Suite1-specific config applies instead of the other config
# because its name comes first according to sort order.
self.assertEqual(
'1-Suite1-specific',
test_aaa.get().overridden_anomaly_config.string_id())
# The 2-Suite1-specific config applies to the other tests under Suite1.
self.assertEqual(
'2-Suite1-general',
test_bbb.get().overridden_anomaly_config.string_id())
self.assertEqual(
'2-Suite1-general',
test_ccc.get().overridden_anomaly_config.string_id())
# Note that Suite2/ddd has no config, and nor do the parent tests.
self.assertIsNone(test_ddd.get().overridden_anomaly_config)
self.assertIsNone(suite1.get().overridden_anomaly_config)
self.assertIsNone(suite2.get().overridden_anomaly_config)
