def testGetBotNamesFromAlerts_RemovesDuplicates(self):
"""Tests that duplicates are removed from the result."""
testing_common.AddTests(['SuperGPU'], ['Bot1'], {'foo': {'bar': {}}})
anomaly.Anomaly(test=utils.TestKey('SuperGPU/Bot1/foo/bar')).put()
anomaly.Anomaly(test=utils.TestKey('SuperGPU/Bot1/foo/bar')).put()
anomalies = anomaly.Anomaly.query().fetch()
bot_names = alert.GetBotNamesFromAlerts(anomalies)
self.assertEqual(2, len(anomalies))
self.assertEqual(1, len(bot_names))
def testGetBugLabelsForTest(self):
bug_label_patterns.AddBugLabelPattern('foo', '*/*/foo')
bug_label_patterns.AddBugLabelPattern('f-prefix', '*/*/f*')
testing_common.AddTests(['M'], ['b'], {'foo': {}, 'bar': {}})
foo_test = utils.TestKey('M/b/foo').get()
bar_test = utils.TestKey('M/b/bar').get()
self.assertEqual(
['f-prefix', 'foo'], bug_label_patterns.GetBugLabelsForTest(foo_test))
self.assertEqual(
[], bug_label_patterns.GetBugLabelsForTest(bar_test))
def testMasterToBotsToDeprecatedDict(self):
self._AddSampleData()
suites = [
utils.TestKey('Chromium/mac/dromaeo').get(),
utils.TestKey('Chromium/win7/dromaeo').get(),
]
suites[0].deprecated = True
suites[0].put()
self.assertEqual(
{'Chromium': {'mac': True, 'win7': False}},
update_test_suites._MasterToBotsToDeprecatedDict(suites))
def testGetBotNamesFromAlerts_TypicalCase(self):
"""Tests that we can get the name of the bots for a list of anomalies."""
testing_common.AddTests(['SuperGPU'], ['Bot1', 'Bot2', 'Bot3'],
{'foo': {
'bar': {}
}})
anomaly.Anomaly(test=utils.TestKey('SuperGPU/Bot1/foo/bar')).put()
anomaly.Anomaly(test=utils.TestKey('SuperGPU/Bot2/foo/bar')).put()
anomaly.Anomaly(test=utils.TestKey('SuperGPU/Bot3/foo/bar')).put()
anomalies = anomaly.Anomaly.query().fetch()
bot_names = alert.GetBotNamesFromAlerts(anomalies)
self.assertEqual({'Bot1', 'Bot2', 'Bot3'}, bot_names)
def testGetSubTests_ReturnsOnlyNonDeprecatedTests(self):
# Sub-tests with the same name may be deprecated on only one bot, and not
# deprecated on another bot; only non-deprecated tests should be returned.
self._AddSampleData()
# Set the deprecated flag to True for one test on one platform.
test = utils.TestKey('Chromium/mac/dromaeo/jslib').get()
test.deprecated = True
test.put()
# Set the has_rows flag to true for all of the test entities.
for test_path in [
'Chromium/win7/dromaeo/dom',
'Chromium/win7/dromaeo/jslib',
'Chromium/mac/dromaeo/dom',
'Chromium/mac/dromaeo/jslib']:
test = utils.TestKey(test_path).get()
test.has_rows = True
test.put()
# When a request is made for subtests for the platform wherein that a
# subtest is deprecated, that subtest will not be listed.
response = self.testapp.post('/list_tests', {
'type': 'sub_tests',
'suite': 'dromaeo',
'bots': 'Chromium/mac'})
self.assertEqual('*', response.headers.get('Access-Control-Allow-Origin'))
expected = {
'dom': {
'has_rows': True,
'sub_tests': {}
},
}
self.assertEqual(expected, json.loads(response.body))
# When a request is made for subtests for multiple platforms, all subtests
# that are not deprecated for at least one of the platforms will be listed.
response = self.testapp.post('/list_tests', {
'type': 'sub_tests',
'suite': 'dromaeo',
'bots': 'Chromium/mac,Chromium/win7'})
self.assertEqual('*', response.headers.get('Access-Control-Allow-Origin'))
expected = {
'dom': {
'has_rows': True,
'sub_tests': {}
},
'jslib': {
'has_rows': True,
'sub_tests': {}
}
}
self.assertEqual(expected, json.loads(response.body))
def _AddTestData(self,
series,
sheriff_key,
improvement_direction=anomaly.UP):
"""Adds one sample Test and associated data.
Args:
series: Either a list of values, or a list of (x, y) pairs.
sheriff_key: A Sheriff entity key.
improvement_direction: One of {anomaly.UP, anomaly.DOWN, anomaly.UNKNOWN}.
Returns:
The Test entity key of the Test that was added.
"""
testing_common.AddTests(['M'], ['b'], {'benchmark': {'t': {}}})
test_path = 'M/b/benchmark/t'
test = utils.TestKey(test_path).get()
test.improvement_direction = improvement_direction
test.sheriff = sheriff_key
sheriff_entity = sheriff_key.get()
sheriff_entity.patterns.append(test.test_path)
sheriff_entity.put()
if series and isinstance(series[0], (int, float)):
series = enumerate(series, start=1)
testing_common.AddRows(test_path, {x: {'value': y} for x, y in series})
return test.put()
def testAddRowsToCache(self):
self._AddMockData()
rows = []
stored_object.Set(
'externally_visible__num_revisions_ChromiumPerf/win7/dromaeo/dom',
[[10, 2, 3], [15, 4, 5], [100, 6, 7]])
test_key = utils.TestKey('ChromiumPerf/win7/dromaeo/dom')
test_container_key = utils.GetTestContainerKey(test_key)
ts1 = datetime.datetime(2013, 1, 1)
row1 = graph_data.Row(parent=test_container_key,
id=1, value=9, timestamp=ts1)
rows.append(row1)
ts2 = datetime.datetime(2013, 1, 2)
row2 = graph_data.Row(parent=test_container_key,
id=12, value=90, timestamp=ts2)
rows.append(row2)
ts3 = datetime.datetime(2013, 1, 3)
row3 = graph_data.Row(parent=test_container_key,
id=102, value=99, timestamp=ts3)
rows.append(row3)
graph_revisions.AddRowsToCache(rows)
self.assertEqual(
[[1, 9, utils.TimestampMilliseconds(ts1)],
[10, 2, 3],
[12, 90, utils.TimestampMilliseconds(ts2)],
[15, 4, 5], [100, 6, 7],
[102, 99, utils.TimestampMilliseconds(ts3)]],
stored_object.Get('externally_visible__num_revisions_'
'ChromiumPerf/win7/dromaeo/dom'))
def testProcessTest_InternalOnlyTest(self, mock_email_sheriff):
self._AddDataForTests()
test = utils.TestKey(
'ChromiumGPU/linux-release/scrolling_benchmark/ref').get()
test.internal_only = True
sheriff.Sheriff(email='*****@*****.**',
id='sheriff',
patterns=[test.test_path]).put()
test.put()
find_anomalies.ProcessTest(test.key)
expected_calls = [
mock.call(
ModelMatcher('sheriff'),
ModelMatcher(
'ChromiumGPU/linux-release/scrolling_benchmark/ref'),
EndRevisionMatcher(10011))
]
self.assertEqual(expected_calls, mock_email_sheriff.call_args_list)
anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(len(anomalies), 1)
self.assertEqual(test.key, anomalies[0].test)
self.assertEqual(100, anomalies[0].percent_changed)
self.assertEqual(anomaly.UP, anomalies[0].direction)
self.assertEqual(10007, anomalies[0].start_revision)
self.assertEqual(10011, anomalies[0].end_revision)
self.assertTrue(anomalies[0].internal_only)
def testPost_MigratesStoppageAlerts(self):
testing_common.AddTests(['Master'], ['b'], {'suite': {'foo': {}}})
test_path = 'Master/b/suite/foo'
test_key = utils.TestKey(test_path)
test_container_key = utils.GetTestContainerKey(test_key)
row_key = graph_data.Row(id=100, parent=test_container_key,
value=5).put()
stoppage_alert.CreateStoppageAlert(test_key.get(), row_key.get()).put()
self.assertIsNotNone(
stoppage_alert.GetStoppageAlert('Master/b/suite/foo', 100))
self.assertIsNone(
stoppage_alert.GetStoppageAlert('Master/b/suite/bar', 100))
self.testapp.post(
'/migrate_test_names', {
'old_pattern': 'Master/b/suite/foo',
'new_pattern': 'Master/b/suite/bar',
})
self.ExecuteTaskQueueTasks('/migrate_test_names',
migrate_test_names._TASK_QUEUE_NAME)
self.assertIsNotNone(
stoppage_alert.GetStoppageAlert('Master/b/suite/bar', 100))
self.assertIsNone(
stoppage_alert.GetStoppageAlert('Master/b/suite/foo', 100))
def _AssertNotExists(self, test_paths):
for test_path in test_paths:
test_key = utils.TestKey(test_path)
num_rows = graph_data.Row.query(
graph_data.Row.parent_test == utils.OldStyleTestKey(test_key)).count()
self.assertEqual(0, num_rows)
self.assertIsNone(test_key.get())
def testPost_DeleteMonitoredTestNotifyFalse_DoesNotSendEmail(self):
self._AddMockData()
# Add a sheriff for one test.
test_path = 'ChromiumPerf/mac/SunSpider/Total/t'
test = utils.TestKey(test_path).get()
sheriff_key = sheriff.Sheriff(id='Perf Sheriff Mac',
email='*****@*****.**',
patterns=['*/*/*/*/*']).put()
test.sheriff = sheriff_key
test.put()
self.testapp.post('/delete_test_data', {
'pattern': 'ChromiumPerf/mac/SunSpider/Total/t',
'notify': 'false',
})
self.ExecuteTaskQueueTasks('/delete_test_data',
delete_test_data._TASK_QUEUE_NAME)
self._AssertNotExists([
'ChromiumPerf/mac/SunSpider/Total/t',
])
# Check the emails that were sent.
messages = self.mail_stub.get_sent_messages()
self.assertEqual(0, len(messages))
def testPost_DeleteMonitoredTest_SendsEmail(self):
self._AddMockData()
# Add a sheriff for one test.
test_path = 'ChromiumPerf/mac/SunSpider/Total/t'
test = utils.TestKey(test_path).get()
sheriff_key = sheriff.Sheriff(id='Perf Sheriff Mac',
email='*****@*****.**',
patterns=['*/*/*/*/*']).put()
test.sheriff = sheriff_key
test.put()
self.testapp.post('/delete_test_data', {
'pattern': 'ChromiumPerf/mac/SunSpider/Total/t',
})
self.ExecuteTaskQueueTasks('/delete_test_data',
delete_test_data._TASK_QUEUE_NAME)
self._AssertNotExists([
'ChromiumPerf/mac/SunSpider/Total/t',
])
# Check the emails that were sent.
messages = self.mail_stub.get_sent_messages()
self.assertEqual(1, len(messages))
self.assertEqual('*****@*****.**', messages[0].sender)
self.assertEqual('*****@*****.**',
messages[0].to)
self.assertEqual('Sheriffed Test Deleted', messages[0].subject)
body = str(messages[0].body)
self.assertIn(
'test ChromiumPerf/mac/SunSpider/Total/t has been DELETED', body)
def testGet_WithAncestor_AllAlertsUpdated(self):
testing_common.AddTests(['M'], ['b1', 'b2'],
{'foo': {
'bar': {},
'baz': {}
}})
testing_common.AddRows(
'M/b1/foo/bar', {i
for i in range(1431001000, 1432001000, 6000)})
test_key = utils.TestKey('M/b1/foo/bar')
# range(1431001000, 1431081000, 6000) includes 14 numbers.
for i in range(1431001000, 1431081000, 6000):
anomaly.Anomaly(start_revision=i,
end_revision=i + 12000,
test=test_key,
median_before_anomaly=100,
median_after_anomaly=200).put()
self.testapp.post('/shrink_timestamp_revisions', {'ancestor': 'M'})
self.ExecuteTaskQueueTasks('/shrink_timestamp_revisions',
shrink_timestamp_revisions._QUEUE_NAME)
anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(14, len(anomalies))
for a in anomalies:
self.assertLess(a.start_revision, 300000)
self.assertLess(a.end_revision, 300000)
def testGetOldStdioUri_InternalOnly_NoURIReturned(self):
testing_common.AddTests(['Master'], ['b'], {'my_suite': {}})
test = utils.TestKey('Master/b/my_suite').get()
test.buildername = 'MyBuilder'
row = graph_data.Row(id=345, buildnumber=456)
test.masterid = 'my.master.id'
test.internal_only = True
self.assertIsNone(graph_json._GetOldStdioUri(row, test))
def testGet_DumpJson_WithRows(self):
# Insert a test with rows.
testing_common.AddTests('M', 'b', {'foo': {}})
test_key = utils.TestKey('M/b/foo')
test_container_key = utils.GetTestContainerKey(test_key)
rows = []
# The upper limit for revision numbers in this test; this was added
# so that the test doesn't depend on the value of _DEFAULT_MAX_POINTS.
highest_rev = 2000 + dump_graph_json._DEFAULT_MAX_POINTS - 1
for rev in range(1000, highest_rev + 1):
row = graph_data.Row(parent=test_container_key,
id=rev,
value=(rev * 2))
rows.append(row)
ndb.put_multi(rows)
# There is a maximum number of rows returned by default, and the rows
# are listed with latest revisions first.
response = self.testapp.get('/dump_graph_json',
{'test_path': 'M/b/foo'})
protobuf_strings = json.loads(response.body)
entities = map(dump_graph_json.BinaryProtobufToEntity,
protobuf_strings)
out_rows = _EntitiesOfKind(entities, 'Row')
expected_num_rows = dump_graph_json._DEFAULT_MAX_POINTS
self.assertEqual(expected_num_rows, len(out_rows))
expected_rev_range = range(highest_rev,
highest_rev + 1 - expected_num_rows, -1)
for expected_rev, row in zip(expected_rev_range, out_rows):
self.assertEqual(expected_rev, row.revision)
self.assertEqual(expected_rev * 2, row.value)
# Specifying end_rev sets the final revision.
response = self.testapp.get('/dump_graph_json', {
'test_path': 'M/b/foo',
'end_rev': 1199
})
protobuf_strings = json.loads(response.body)
entities = map(dump_graph_json.BinaryProtobufToEntity,
protobuf_strings)
out_rows = _EntitiesOfKind(entities, 'Row')
expected_num_rows = min(dump_graph_json._DEFAULT_MAX_POINTS, 200)
self.assertEqual(expected_num_rows, len(out_rows))
self.assertEqual(1199, out_rows[0].revision)
# An alternative max number of rows can be specified.
response = self.testapp.get('/dump_graph_json', {
'test_path': 'M/b/foo',
'num_points': 4
})
protobuf_strings = json.loads(response.body)
entities = map(dump_graph_json.BinaryProtobufToEntity,
protobuf_strings)
out_rows = _EntitiesOfKind(entities, 'Row')
rev_nums = [row.revision for row in out_rows]
expected_rev_range = range(highest_rev, highest_rev - 4, -1)
self.assertEqual(expected_rev_range, rev_nums)
def testPointInfoDict_RowHasNoTracingUri_ResultHasNoTracingUri(self):
testing_common.AddTests(['Master'], ['b'], {'my_suite': {}})
test = utils.TestKey('Master/b/my_suite').get()
rows = testing_common.AddRows('Master/b/my_suite', [345])
# This row has no a_tracing_uri property, so there should be no
# trace annotation returned by _PointInfoDict.
point_info = graph_json._PointInfoDict(rows[0], test, {})
self.assertFalse(hasattr(rows[0], 'a_tracing_uri'))
self.assertNotIn('a_tracing_uri', point_info)
def testCreateSuiteMonitoredDict(self):
self._AddSampleData()
test_win = utils.TestKey('Chromium/win7/dromaeo').get()
test_win.monitored = [utils.TestKey(
'Chromium/win7/dromaeo/commit_time/www.yahoo.com')]
test_win.put()
test_mac = utils.TestKey('Chromium/mac/dromaeo').get()
test_mac.monitored = [utils.TestKey(
'Chromium/mac/dromaeo/commit_time/www.cnn.com')]
test_mac.put()
self.assertEqual(
{
'dromaeo': [
'commit_time/www.cnn.com',
'commit_time/www.yahoo.com',
]
},
update_test_suites._CreateSuiteMonitoredDict())
