def _AddDataForTests(self, stats=None, masters=None):
if not masters:
masters = ['ChromiumGPU']
testing_common.AddTests(masters, ['linux-release'], {
'scrolling_benchmark': {
'ref': {},
},
})
for m in masters:
ref = utils.TestKey('%s/linux-release/scrolling_benchmark/ref' %
m).get()
ref.units = 'ms'
for i in range(9000, 10070, 5):
# Internal-only data should be found.
test_container_key = utils.GetTestContainerKey(ref.key)
r = graph_data.Row(id=i + 1,
value=float(i * 3),
parent=test_container_key,
internal_only=True)
if stats:
for s in stats:
setattr(r, s, i)
r.put()
def testAddRowsToCache(self):
self._AddMockData()
rows = []
stored_object.Set(
'externally_visible__num_revisions_ChromiumPerf/win7/dromaeo/dom',
[[10, 2, 3], [15, 4, 5], [100, 6, 7]])
test_key = utils.TestKey('ChromiumPerf/win7/dromaeo/dom')
test_container_key = utils.GetTestContainerKey(test_key)
ts1 = datetime.datetime(2013, 1, 1)
row1 = graph_data.Row(parent=test_container_key,
id=1,
value=9,
timestamp=ts1)
rows.append(row1)
ts2 = datetime.datetime(2013, 1, 2)
row2 = graph_data.Row(parent=test_container_key,
id=12,
value=90,
timestamp=ts2)
rows.append(row2)
ts3 = datetime.datetime(2013, 1, 3)
row3 = graph_data.Row(parent=test_container_key,
id=102,
value=99,
timestamp=ts3)
rows.append(row3)
graph_revisions.AddRowsToCache(rows)
self.assertEqual(
[[1, 9, utils.TimestampMilliseconds(ts1)], [10, 2, 3],
[12, 90, utils.TimestampMilliseconds(ts2)], [15, 4, 5],
[100, 6, 7], [102, 99, utils.TimestampMilliseconds(ts3)]],
stored_object.Get('externally_visible__num_revisions_'
'ChromiumPerf/win7/dromaeo/dom'))
def _AddLongTestColumns(self, start_rev=15000, end_rev=16500, step=3):
"""Adds test data with long nested sub test to the mock datastore.
Args:
start_rev: Starting revision number.
end_rev: Ending revision number.
step: Difference between adjacent revisions.
"""
master = graph_data.Master(id='master')
master.put()
bot = graph_data.Bot(id='bot', parent=master.key)
bot.put()
test = graph_data.TestMetadata(id='master/bot/suite')
test.UpdateSheriff()
test.put()
rows = []
path = 'master/bot/suite'
for sub_name in ['sub1', 'sub2', 'sub3', 'sub4', 'sub5']:
path = '%s/%s' % (path, sub_name)
test = graph_data.TestMetadata(id=path,
improvement_direction=anomaly.UP,
has_rows=True)
test.UpdateSheriff()
test.put()
test_container_key = utils.GetTestContainerKey(test.key)
for i in range(start_rev, end_rev, step):
row = graph_data.Row(parent=test_container_key,
id=i,
value=float(i * 2),
r_webkit=int(i * 0.25),
a_str='some_string',
buildnumber=i - start_rev,
a_tracing_uri='http://trace/%d' % i)
rows.append(row)
ndb.put_multi(rows)
def _AddMockData(self):
"""Adds sample TestMetadata, Row, and Anomaly entities."""
testing_common.AddTests(*_MOCK_DATA)
# Add 50 Row entities to one of the tests.
# Also add 2 Anomaly entities.
test_path = 'ChromiumPerf/mac/SunSpider/Total/t'
test_key = utils.TestKey(test_path)
test_container_key = utils.GetTestContainerKey(test_key)
for rev in range(15000, 15100, 2):
graph_data.Row(id=rev, parent=test_container_key, value=(rev * 2)).put()
if rev % 50 == 0:
data = generic_set.GenericSet(['foo_%s' % rev])
data = data.AsDict()
anomaly.Anomaly(
start_revision=(rev - 2), end_revision=rev,
median_before_anomaly=100, median_after_anomaly=50,
test=test_key).put()
histogram.SparseDiagnostic(
test=test_key,
start_revision=rev - 50, end_revision=rev - 1,
data=data).put()
histogram.Histogram(
test=test_key).put()
def testProcessTest__RefineAnomalyPlacement_BalancedEstimator2(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'],
{'blink_perf.layout': {
'nested-percent-height-tables': {}
}})
test = utils.TestKey(
'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables'
).get()
test_container_key = utils.GetTestContainerKey(test.key)
sample_data = [
(793468, 136.5382),
(793486, 137.7192),
(793495, 137.4038),
(793504, 137.4919),
(793505, 137.4465),
(793518, 136.9279),
(793525, 137.3501),
(793528, 136.9622),
(793543, 137.1027),
(793550, 137.7351),
(793555, 137.1511),
(793559, 137.2094),
(793560, 136.5192),
(793565, 138.1536),
(793580, 137.4172),
(793590, 136.8746),
(793601, 137.5016),
(793609, 137.0773),
(793625, 137.4702),
(793646, 135.9019),
(793657, 137.2827),
(793702, 136.5978),
(793712, 136.0732),
(793721, 132.1820),
(793742, 122.1631),
(793760, 136.3152),
(793774, 136.9616),
(793788, 136.8438),
(794016, 136.3022),
(794024, 136.3495),
(794027, 136.3145),
(794036, 136.5502),
(794043, 136.3861),
(794051, 136.2035),
(794059, 136.2348),
(794066, 136.2594),
(794074, 135.9686),
(794088, 136.7375),
(794107, 136.5570),
(794132,
129.9924), # This one is a potential change point - but weak
(794143, 135.8275),
(794154, 107.2502), # This is a better change point
(794158, 108.3948),
(794160, 107.3564),
(794196, 107.9707),
(794236, 111.3168),
(794268, 108.7905),
(794281, 111.1065),
(794319, 109.7699),
(794320, 109.8082),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.DOWN, new_anomalies[0].direction)
self.assertEqual(794144, new_anomalies[0].start_revision)
self.assertEqual(794154, new_anomalies[0].end_revision)
def testProcessTest__RefineAnomalyPlacement_BalancedEstimator1(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'],
{'blink_perf.layout': {
'nested-percent-height-tables': {}
}})
test = utils.TestKey(
'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables'
).get()
test_container_key = utils.GetTestContainerKey(test.key)
sample_data = [
(818289, 2009771),
(818290, 1966080),
(818291, 1966080),
(818293, 1966080),
(818294, 2053461),
(818296, 2009771),
(818298, 1966080),
(818301, 2009771),
(818303, 2009771),
(818305, 2009771),
(818306, 2009771),
(818307, 1966080),
(818308, 2009771),
(818309, 2009771),
(818310, 1966080),
(818311, 2009771),
(818312, 1966080),
(818317, 1966080),
(818318, 1966080),
(818320, 2053461),
(818322, 2009771),
(818326, 1966080),
(818331, 1966080),
(818335, 1966080),
(818340, 2009771),
(818347, 2009771),
(818350, 1966080),
(818353, 1966080),
(818354, 2009771),
(818361, 2009771),
(818362, 1966080),
(818374, 2009771),
(818379, 2009771),
(818382, 2053461),
(818389, 2009771),
(818402, 1966080),
(818409, 2009771),
(818416, 1966080),
(818420, 1966080),
(818430, 2009771),
(818440, 2228224),
(818450, 2228224),
(818461, 2228224),
(818469, 2228224),
(818481, 2228224),
(818498, 2271915),
(818514, 2228224),
(818531, 2271915),
(818571, 2271915),
(818583, 2271915),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.UP, new_anomalies[0].direction)
self.assertEqual(818431, new_anomalies[0].start_revision)
self.assertEqual(818440, new_anomalies[0].end_revision)
def testProcessTest_MultipleChangePoints(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'],
{'blink_perf.layout': {
'nested-percent-height-tables': {}
}})
test = utils.TestKey(
'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables'
).get()
test_container_key = utils.GetTestContainerKey(test.key)
sample_data = [
(804863, 13830765),
(804867, 16667862),
(804879, 13929296),
(804891, 13823876),
(804896, 13908794),
(804900, 13899281),
(804907, 14901462),
(804921, 13890597),
(804935, 13969113),
(804946, 13996520),
(804957, 13913104),
(805143, 16770364),
(805175, 14858529),
(805179, 14013942),
(805185, 14857516),
(805195, 14895168),
(805196, 14944037),
(805205, 13919484),
(805211, 15736581),
(805231, 14730142),
(805236, 13892102),
(805247, 14808876),
(805253, 14903648),
(805262, 13896626),
(805276, 15797878),
(805281, 14542593),
(805285, 15733168),
(805290, 13882841),
(805302, 15727394),
(805314, 15758058),
(805333, 16074960),
(805345, 16142162),
(805359, 16138912),
(805384, 17914289),
(805412, 18368834),
(805428, 18055197),
(805457, 19673614),
(805482, 19705606),
(805502, 19609089),
(805509, 19576745),
(805531, 19600059),
(805550, 19702969),
(805564, 19660953),
(805584, 19830273),
(805600, 19800662),
(805606, 19493150),
(805620, 19700545),
(805624, 19623731),
(805628, 19683921),
(805634, 19660001),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(2, len(new_anomalies))
self.assertEqual(anomaly.UP, new_anomalies[0].direction)
self.assertEqual(805429, new_anomalies[0].start_revision)
self.assertEqual(805457, new_anomalies[0].end_revision)
self.assertEqual(805315, new_anomalies[1].start_revision)
self.assertEqual(805428, new_anomalies[1].end_revision)
def testProcessTest_RefineAnomalyPlacement_OffByOneStable(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'], {
'memory.desktop': {
('memory:chrome:all_processes:'
'reported_by_chrome:v8:effective_size_avg'): {}
}
})
test = utils.TestKey((
'ChromiumPerf/linux-perf/memory.desktop/'
'memory:chrome:all_processes:reported_by_chrome:v8:effective_size_avg'
)).get()
test_container_key = utils.GetTestContainerKey(test.key)
sample_data = [
(733480, 1381203.0),
(733494, 1381220.0),
(733504, 1381212.0),
(733524, 1381220.0),
(733538, 1381211.0),
(733544, 1381212.0),
(733549, 1381220.0),
(733563, 1381220.0),
(733581, 1381220.0),
(733597, 1381212.0),
(733611, 1381228.0),
(733641, 1381212.0),
(733675, 1381204.0),
(733721, 1381212.0),
(733766, 1381211.0),
(733804, 1381204.0),
(733835, 1381219.0),
(733865, 1381211.0),
(733885, 1381219.0),
(733908, 1381204.0),
(733920, 1381211.0),
(733937, 1381220.0),
(734091, 1381211.0),
(734133, 1381219.0),
(734181, 1381204.0),
(734211, 1381720.0),
(734248, 1381712.0),
(734277, 1381696.0),
(734311, 1381704.0),
(734341, 1381703.0),
(734372, 1381704.0),
(734405, 1381703.0),
(734431, 1381711.0),
(734456, 1381720.0),
(734487, 1381703.0),
(734521, 1381704.0),
(734554, 1381726.0),
(734598, 1381704.0),
(734630,
1381703.0), # In crbug/1041688 this is where it was placed.
(734673, 1529888.0), # This is where it should be.
(734705, 1529888.0),
(734739, 1529860.0),
(734770, 1529860.0),
(734793, 1529888.0),
(734829, 1529860.0),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.UP, new_anomalies[0].direction)
self.assertEqual(734631, new_anomalies[0].start_revision)
self.assertEqual(734673, new_anomalies[0].end_revision)
def testProcessTest_RefineAnomalyPlacement_OffByOneBefore(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'],
{'blink_perf.layout': {
'nested-percent-height-tables': {}
}})
test = utils.TestKey(
'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables'
).get()
test_container_key = utils.GetTestContainerKey(test.key)
sample_data = [
(728446, 480.2504),
(728462, 487.685),
(728469, 486.6389),
(728480, 477.6597),
(728492, 471.2238),
(728512, 480.4379),
(728539, 464.5573),
(728594, 489.0594),
(728644, 484.4796),
(728714, 489.5986),
(728751, 489.474),
(728788, 481.9336),
(728835, 484.089),
(728869, 485.4287),
(728883, 476.8234),
(728907, 487.4736),
(728938, 490.601),
(728986, 483.5039),
(729021, 485.176),
(729066, 484.5855),
(729105, 483.9114),
(729119, 483.559),
(729161, 477.6875),
(729201, 484.9668),
(729240, 480.7091),
(729270, 484.5506),
(729292, 495.1445),
(729309, 479.9111),
(729329, 479.8815),
(729391, 487.5683),
(729430, 476.7355),
(729478, 487.7251),
(729525, 493.1012),
(729568, 497.7565),
(729608, 499.6481),
(729642, 496.1591),
(729658, 493.4581),
(729687, 486.1097),
(729706, 478.036),
(729730,
480.4222), # In crbug/1041688 this was the original placement.
(729764, 421.0342), # We instead should be setting it here.
(729795, 428.0284),
(729846, 433.8261),
(729883, 429.49),
(729920, 436.3342),
(729975, 434.3996),
(730011, 428.3672),
(730054, 436.309),
(730094, 435.3792),
(730128, 433.0537),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
test.UpdateSheriff()
test.put()
with mock.patch.object(SheriffConfigClient, 'Match',
mock.MagicMock(return_value=([], None))) as m:
find_anomalies.ProcessTests([test.key])
self.assertEqual(m.call_args_list, [mock.call(test.test_path)])
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(1, len(new_anomalies))
self.assertEqual(anomaly.DOWN, new_anomalies[0].direction)
self.assertEqual(729731, new_anomalies[0].start_revision)
self.assertEqual(729764, new_anomalies[0].end_revision)
def testProcessTest__RefineAnomalyPlacement_OnePassEDivisive(self):
testing_common.AddTests(
['ChromiumPerf'], ['linux-perf'],
{'blink_perf.layout': {
'nested-percent-height-tables': {}
}})
test = utils.TestKey(
'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables'
).get()
test_container_key = utils.GetTestContainerKey(test.key)
# 1608562683 will be anomaly if we run E-Divisive from 1608525044.
sample_data = [
(1608404024, 246272),
(1608407660, 249344),
(1608417360, 246272),
(1608422547, 246784),
(1608434678, 248832),
(1608440108, 248320),
(1608442260, 250880),
(1608452306, 248832),
(1608457404, 247296),
(1608459374, 247296),
(1608463502, 249344),
(1608469894, 247296),
(1608471945, 247296),
(1608477313, 246272),
(1608481014, 248832),
(1608484511, 247296),
(1608486532, 246784),
(1608488082, 248832),
(1608491972, 246784),
(1608493895, 248832),
(1608495366, 248320),
(1608498927, 252416),
(1608501293, 246784),
(1608505924, 246272),
(1608507885, 246784),
(1608509593, 250368),
(1608512971, 246784),
(1608515075, 246272),
(1608519889, 247296),
(1608521956, 254464),
(1608525044, 247296),
(1608526992, 244736),
(1608528640, 245760),
(1608530391, 246784),
(1608531986, 245760),
(1608533763, 245760),
(1608538109, 246272),
(1608539988, 246784),
(1608545280, 251392),
(1608547200, 251026),
(1608550736, 248320),
(1608552820, 248832),
(1608554780, 251392),
(1608560589, 247296),
(1608562683, 251904),
(1608564319, 268800),
(1608566089, 263168),
(1608567823, 266240),
(1608569370, 266752),
(1608570921, 264192),
]
for row in sample_data:
graph_data.Row(id=row[0], value=row[1],
parent=test_container_key).put()
test.UpdateSheriff()
test.put()
new_anomalies = anomaly.Anomaly.query().fetch()
self.assertEqual(0, len(new_anomalies))
def row(self):
test_container = utils.GetTestContainerKey(self.GetTestMetadataKey())
return ndb.Key('Row', self.revision, parent=test_container)
def _AddDataToDatastore(self):
"""Puts a set of entities; some internal-only, some not."""
# Need to be privileged to add TestMetadata and Row objects to the datastore
# because there is a get() for the parent_test in the pre_put_hook. This
# should work correctly in production because Rows and TestMetadata should
# only be added by /add_point, which is privileged.
self.SetCurrentUser('*****@*****.**')
testing_common.AddTests(
['ChromiumPerf'], ['Win7External', 'FooInternal'], {
'TestInternal': {
'SubTestInternal': {}
},
'TestExternal': {
'SubTestExternal': {}
},
})
internal_key = ['Master', 'ChromiumPerf', 'Bot', 'FooInternal']
internal_bot = ndb.Key(*internal_key).get()
internal_bot.internal_only = True
internal_bot.put()
internal_test = ndb.Key(
'TestMetadata', 'ChromiumPerf/Win7External/TestInternal').get()
internal_test.internal_only = True
internal_test.put()
internal_test = ndb.Key('TestMetadata',
'ChromiumPerf/FooInternal/TestInternal').get()
internal_test.internal_only = True
internal_test.put()
internal_sub_test = ndb.Key(
'TestMetadata',
'ChromiumPerf/Win7External/TestInternal/SubTestInternal').get()
internal_sub_test.internal_only = True
internal_sub_test.put()
internal_sub_test = ndb.Key(
'TestMetadata',
'ChromiumPerf/FooInternal/TestInternal/SubTestInternal').get()
internal_sub_test.internal_only = True
internal_sub_test.put()
internal_key = internal_sub_test.key
external_key = ndb.Key(
'TestMetadata',
'ChromiumPerf/Win7External/TestExternal/SubTestExternal')
internal_test_container_key = utils.GetTestContainerKey(internal_key)
external_test_container_key = utils.GetTestContainerKey(external_key)
for i in range(0, 100, 10):
graph_data.Row(parent=internal_test_container_key,
id=i,
value=float(i * 2),
internal_only=True).put()
graph_data.Row(parent=external_test_container_key,
id=i,
value=float(i * 2)).put()
self.UnsetCurrentUser()
sheriff.Sheriff(id='external',
email='*****@*****.**',
internal_only=False).put()
sheriff.Sheriff(id='internal',
email='*****@*****.**',
internal_only=True).put()