def _AddDataForTests(self, stats=None, masters=None): if not masters: masters = ['ChromiumGPU'] testing_common.AddTests(masters, ['linux-release'], { 'scrolling_benchmark': { 'ref': {}, }, }) for m in masters: ref = utils.TestKey('%s/linux-release/scrolling_benchmark/ref' % m).get() ref.units = 'ms' for i in range(9000, 10070, 5): # Internal-only data should be found. test_container_key = utils.GetTestContainerKey(ref.key) r = graph_data.Row(id=i + 1, value=float(i * 3), parent=test_container_key, internal_only=True) if stats: for s in stats: setattr(r, s, i) r.put()
def testAddRowsToCache(self): self._AddMockData() rows = [] stored_object.Set( 'externally_visible__num_revisions_ChromiumPerf/win7/dromaeo/dom', [[10, 2, 3], [15, 4, 5], [100, 6, 7]]) test_key = utils.TestKey('ChromiumPerf/win7/dromaeo/dom') test_container_key = utils.GetTestContainerKey(test_key) ts1 = datetime.datetime(2013, 1, 1) row1 = graph_data.Row(parent=test_container_key, id=1, value=9, timestamp=ts1) rows.append(row1) ts2 = datetime.datetime(2013, 1, 2) row2 = graph_data.Row(parent=test_container_key, id=12, value=90, timestamp=ts2) rows.append(row2) ts3 = datetime.datetime(2013, 1, 3) row3 = graph_data.Row(parent=test_container_key, id=102, value=99, timestamp=ts3) rows.append(row3) graph_revisions.AddRowsToCache(rows) self.assertEqual( [[1, 9, utils.TimestampMilliseconds(ts1)], [10, 2, 3], [12, 90, utils.TimestampMilliseconds(ts2)], [15, 4, 5], [100, 6, 7], [102, 99, utils.TimestampMilliseconds(ts3)]], stored_object.Get('externally_visible__num_revisions_' 'ChromiumPerf/win7/dromaeo/dom'))
def _AddLongTestColumns(self, start_rev=15000, end_rev=16500, step=3): """Adds test data with long nested sub test to the mock datastore. Args: start_rev: Starting revision number. end_rev: Ending revision number. step: Difference between adjacent revisions. """ master = graph_data.Master(id='master') master.put() bot = graph_data.Bot(id='bot', parent=master.key) bot.put() test = graph_data.TestMetadata(id='master/bot/suite') test.UpdateSheriff() test.put() rows = [] path = 'master/bot/suite' for sub_name in ['sub1', 'sub2', 'sub3', 'sub4', 'sub5']: path = '%s/%s' % (path, sub_name) test = graph_data.TestMetadata(id=path, improvement_direction=anomaly.UP, has_rows=True) test.UpdateSheriff() test.put() test_container_key = utils.GetTestContainerKey(test.key) for i in range(start_rev, end_rev, step): row = graph_data.Row(parent=test_container_key, id=i, value=float(i * 2), r_webkit=int(i * 0.25), a_str='some_string', buildnumber=i - start_rev, a_tracing_uri='http://trace/%d' % i) rows.append(row) ndb.put_multi(rows)
def _AddMockData(self): """Adds sample TestMetadata, Row, and Anomaly entities.""" testing_common.AddTests(*_MOCK_DATA) # Add 50 Row entities to one of the tests. # Also add 2 Anomaly entities. test_path = 'ChromiumPerf/mac/SunSpider/Total/t' test_key = utils.TestKey(test_path) test_container_key = utils.GetTestContainerKey(test_key) for rev in range(15000, 15100, 2): graph_data.Row(id=rev, parent=test_container_key, value=(rev * 2)).put() if rev % 50 == 0: data = generic_set.GenericSet(['foo_%s' % rev]) data = data.AsDict() anomaly.Anomaly( start_revision=(rev - 2), end_revision=rev, median_before_anomaly=100, median_after_anomaly=50, test=test_key).put() histogram.SparseDiagnostic( test=test_key, start_revision=rev - 50, end_revision=rev - 1, data=data).put() histogram.Histogram( test=test_key).put()
def testProcessTest__RefineAnomalyPlacement_BalancedEstimator2(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (793468, 136.5382), (793486, 137.7192), (793495, 137.4038), (793504, 137.4919), (793505, 137.4465), (793518, 136.9279), (793525, 137.3501), (793528, 136.9622), (793543, 137.1027), (793550, 137.7351), (793555, 137.1511), (793559, 137.2094), (793560, 136.5192), (793565, 138.1536), (793580, 137.4172), (793590, 136.8746), (793601, 137.5016), (793609, 137.0773), (793625, 137.4702), (793646, 135.9019), (793657, 137.2827), (793702, 136.5978), (793712, 136.0732), (793721, 132.1820), (793742, 122.1631), (793760, 136.3152), (793774, 136.9616), (793788, 136.8438), (794016, 136.3022), (794024, 136.3495), (794027, 136.3145), (794036, 136.5502), (794043, 136.3861), (794051, 136.2035), (794059, 136.2348), (794066, 136.2594), (794074, 135.9686), (794088, 136.7375), (794107, 136.5570), (794132, 129.9924), # This one is a potential change point - but weak (794143, 135.8275), (794154, 107.2502), # This is a better change point (794158, 108.3948), (794160, 107.3564), (794196, 107.9707), (794236, 111.3168), (794268, 108.7905), (794281, 111.1065), (794319, 109.7699), (794320, 109.8082), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.DOWN, new_anomalies[0].direction) self.assertEqual(794144, new_anomalies[0].start_revision) self.assertEqual(794154, new_anomalies[0].end_revision)
def testProcessTest__RefineAnomalyPlacement_BalancedEstimator1(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (818289, 2009771), (818290, 1966080), (818291, 1966080), (818293, 1966080), (818294, 2053461), (818296, 2009771), (818298, 1966080), (818301, 2009771), (818303, 2009771), (818305, 2009771), (818306, 2009771), (818307, 1966080), (818308, 2009771), (818309, 2009771), (818310, 1966080), (818311, 2009771), (818312, 1966080), (818317, 1966080), (818318, 1966080), (818320, 2053461), (818322, 2009771), (818326, 1966080), (818331, 1966080), (818335, 1966080), (818340, 2009771), (818347, 2009771), (818350, 1966080), (818353, 1966080), (818354, 2009771), (818361, 2009771), (818362, 1966080), (818374, 2009771), (818379, 2009771), (818382, 2053461), (818389, 2009771), (818402, 1966080), (818409, 2009771), (818416, 1966080), (818420, 1966080), (818430, 2009771), (818440, 2228224), (818450, 2228224), (818461, 2228224), (818469, 2228224), (818481, 2228224), (818498, 2271915), (818514, 2228224), (818531, 2271915), (818571, 2271915), (818583, 2271915), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(818431, new_anomalies[0].start_revision) self.assertEqual(818440, new_anomalies[0].end_revision)
def testProcessTest_MultipleChangePoints(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (804863, 13830765), (804867, 16667862), (804879, 13929296), (804891, 13823876), (804896, 13908794), (804900, 13899281), (804907, 14901462), (804921, 13890597), (804935, 13969113), (804946, 13996520), (804957, 13913104), (805143, 16770364), (805175, 14858529), (805179, 14013942), (805185, 14857516), (805195, 14895168), (805196, 14944037), (805205, 13919484), (805211, 15736581), (805231, 14730142), (805236, 13892102), (805247, 14808876), (805253, 14903648), (805262, 13896626), (805276, 15797878), (805281, 14542593), (805285, 15733168), (805290, 13882841), (805302, 15727394), (805314, 15758058), (805333, 16074960), (805345, 16142162), (805359, 16138912), (805384, 17914289), (805412, 18368834), (805428, 18055197), (805457, 19673614), (805482, 19705606), (805502, 19609089), (805509, 19576745), (805531, 19600059), (805550, 19702969), (805564, 19660953), (805584, 19830273), (805600, 19800662), (805606, 19493150), (805620, 19700545), (805624, 19623731), (805628, 19683921), (805634, 19660001), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(2, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(805429, new_anomalies[0].start_revision) self.assertEqual(805457, new_anomalies[0].end_revision) self.assertEqual(805315, new_anomalies[1].start_revision) self.assertEqual(805428, new_anomalies[1].end_revision)
def testProcessTest_RefineAnomalyPlacement_OffByOneStable(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], { 'memory.desktop': { ('memory:chrome:all_processes:' 'reported_by_chrome:v8:effective_size_avg'): {} } }) test = utils.TestKey(( 'ChromiumPerf/linux-perf/memory.desktop/' 'memory:chrome:all_processes:reported_by_chrome:v8:effective_size_avg' )).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (733480, 1381203.0), (733494, 1381220.0), (733504, 1381212.0), (733524, 1381220.0), (733538, 1381211.0), (733544, 1381212.0), (733549, 1381220.0), (733563, 1381220.0), (733581, 1381220.0), (733597, 1381212.0), (733611, 1381228.0), (733641, 1381212.0), (733675, 1381204.0), (733721, 1381212.0), (733766, 1381211.0), (733804, 1381204.0), (733835, 1381219.0), (733865, 1381211.0), (733885, 1381219.0), (733908, 1381204.0), (733920, 1381211.0), (733937, 1381220.0), (734091, 1381211.0), (734133, 1381219.0), (734181, 1381204.0), (734211, 1381720.0), (734248, 1381712.0), (734277, 1381696.0), (734311, 1381704.0), (734341, 1381703.0), (734372, 1381704.0), (734405, 1381703.0), (734431, 1381711.0), (734456, 1381720.0), (734487, 1381703.0), (734521, 1381704.0), (734554, 1381726.0), (734598, 1381704.0), (734630, 1381703.0), # In crbug/1041688 this is where it was placed. (734673, 1529888.0), # This is where it should be. (734705, 1529888.0), (734739, 1529860.0), (734770, 1529860.0), (734793, 1529888.0), (734829, 1529860.0), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(734631, new_anomalies[0].start_revision) self.assertEqual(734673, new_anomalies[0].end_revision)
def testProcessTest_RefineAnomalyPlacement_OffByOneBefore(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (728446, 480.2504), (728462, 487.685), (728469, 486.6389), (728480, 477.6597), (728492, 471.2238), (728512, 480.4379), (728539, 464.5573), (728594, 489.0594), (728644, 484.4796), (728714, 489.5986), (728751, 489.474), (728788, 481.9336), (728835, 484.089), (728869, 485.4287), (728883, 476.8234), (728907, 487.4736), (728938, 490.601), (728986, 483.5039), (729021, 485.176), (729066, 484.5855), (729105, 483.9114), (729119, 483.559), (729161, 477.6875), (729201, 484.9668), (729240, 480.7091), (729270, 484.5506), (729292, 495.1445), (729309, 479.9111), (729329, 479.8815), (729391, 487.5683), (729430, 476.7355), (729478, 487.7251), (729525, 493.1012), (729568, 497.7565), (729608, 499.6481), (729642, 496.1591), (729658, 493.4581), (729687, 486.1097), (729706, 478.036), (729730, 480.4222), # In crbug/1041688 this was the original placement. (729764, 421.0342), # We instead should be setting it here. (729795, 428.0284), (729846, 433.8261), (729883, 429.49), (729920, 436.3342), (729975, 434.3996), (730011, 428.3672), (730054, 436.309), (730094, 435.3792), (730128, 433.0537), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.DOWN, new_anomalies[0].direction) self.assertEqual(729731, new_anomalies[0].start_revision) self.assertEqual(729764, new_anomalies[0].end_revision)
def testProcessTest__RefineAnomalyPlacement_OnePassEDivisive(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) # 1608562683 will be anomaly if we run E-Divisive from 1608525044. sample_data = [ (1608404024, 246272), (1608407660, 249344), (1608417360, 246272), (1608422547, 246784), (1608434678, 248832), (1608440108, 248320), (1608442260, 250880), (1608452306, 248832), (1608457404, 247296), (1608459374, 247296), (1608463502, 249344), (1608469894, 247296), (1608471945, 247296), (1608477313, 246272), (1608481014, 248832), (1608484511, 247296), (1608486532, 246784), (1608488082, 248832), (1608491972, 246784), (1608493895, 248832), (1608495366, 248320), (1608498927, 252416), (1608501293, 246784), (1608505924, 246272), (1608507885, 246784), (1608509593, 250368), (1608512971, 246784), (1608515075, 246272), (1608519889, 247296), (1608521956, 254464), (1608525044, 247296), (1608526992, 244736), (1608528640, 245760), (1608530391, 246784), (1608531986, 245760), (1608533763, 245760), (1608538109, 246272), (1608539988, 246784), (1608545280, 251392), (1608547200, 251026), (1608550736, 248320), (1608552820, 248832), (1608554780, 251392), (1608560589, 247296), (1608562683, 251904), (1608564319, 268800), (1608566089, 263168), (1608567823, 266240), (1608569370, 266752), (1608570921, 264192), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(0, len(new_anomalies))
def row(self): test_container = utils.GetTestContainerKey(self.GetTestMetadataKey()) return ndb.Key('Row', self.revision, parent=test_container)
def _AddDataToDatastore(self): """Puts a set of entities; some internal-only, some not.""" # Need to be privileged to add TestMetadata and Row objects to the datastore # because there is a get() for the parent_test in the pre_put_hook. This # should work correctly in production because Rows and TestMetadata should # only be added by /add_point, which is privileged. self.SetCurrentUser('*****@*****.**') testing_common.AddTests( ['ChromiumPerf'], ['Win7External', 'FooInternal'], { 'TestInternal': { 'SubTestInternal': {} }, 'TestExternal': { 'SubTestExternal': {} }, }) internal_key = ['Master', 'ChromiumPerf', 'Bot', 'FooInternal'] internal_bot = ndb.Key(*internal_key).get() internal_bot.internal_only = True internal_bot.put() internal_test = ndb.Key( 'TestMetadata', 'ChromiumPerf/Win7External/TestInternal').get() internal_test.internal_only = True internal_test.put() internal_test = ndb.Key('TestMetadata', 'ChromiumPerf/FooInternal/TestInternal').get() internal_test.internal_only = True internal_test.put() internal_sub_test = ndb.Key( 'TestMetadata', 'ChromiumPerf/Win7External/TestInternal/SubTestInternal').get() internal_sub_test.internal_only = True internal_sub_test.put() internal_sub_test = ndb.Key( 'TestMetadata', 'ChromiumPerf/FooInternal/TestInternal/SubTestInternal').get() internal_sub_test.internal_only = True internal_sub_test.put() internal_key = internal_sub_test.key external_key = ndb.Key( 'TestMetadata', 'ChromiumPerf/Win7External/TestExternal/SubTestExternal') internal_test_container_key = utils.GetTestContainerKey(internal_key) external_test_container_key = utils.GetTestContainerKey(external_key) for i in range(0, 100, 10): graph_data.Row(parent=internal_test_container_key, id=i, value=float(i * 2), internal_only=True).put() graph_data.Row(parent=external_test_container_key, id=i, value=float(i * 2)).put() self.UnsetCurrentUser() sheriff.Sheriff(id='external', email='*****@*****.**', internal_only=False).put() sheriff.Sheriff(id='internal', email='*****@*****.**', internal_only=True).put()