def testInsertOrReplaceRecords_existingTable(self): columns = ('bug_id', 'summary', 'status') df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'), (456, 'Another bug', 'Assigned')], columns=columns, index=columns[0]) df2 = pandas.DataFrame.from_records([(123, 'Some bug', 'Fixed'), (789, 'A new bug', 'Untriaged')], columns=columns, index=columns[0]) conn = sqlite3.connect(':memory:') try: # Write first data frame to database. pandas_sqlite.InsertOrReplaceRecords(df1, 'bugs', conn) df = pandas.read_sql('SELECT * FROM bugs', conn, index_col=columns[0]) self.assertEqual(len(df), 2) self.assertEqual(df.loc[123]['status'], 'Started') # Write second data frame to database. pandas_sqlite.InsertOrReplaceRecords(df2, 'bugs', conn) df = pandas.read_sql('SELECT * FROM bugs', conn, index_col=columns[0]) self.assertEqual(len(df), 3) # Only one extra record added. self.assertEqual(df.loc[123]['status'], 'Fixed') # Bug is now fixed. self.assertItemsEqual(df.index, (123, 456, 789)) finally: conn.close()
def testInsertOrReplaceRecords_existingRecords(self): column_types = (('bug_id', int), ('summary', str), ('status', str)) rows1 = [(123, 'Some bug', 'Started'), (456, 'Another bug', 'Assigned')] df1 = pandas_sqlite.DataFrame(column_types, index='bug_id', rows=rows1) rows2 = [(123, 'Some bug', 'Fixed'), (789, 'A new bug', 'Untriaged')] df2 = pandas_sqlite.DataFrame(column_types, index='bug_id', rows=rows2) con = sqlite3.connect(':memory:') try: pandas_sqlite.CreateTableIfNotExists(con, 'bugs', df1) # Write first data frame to database. pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df1) df = pandas.read_sql('SELECT * FROM bugs', con, index_col='bug_id') self.assertEqual(len(df), 2) self.assertEqual(df.loc[123]['status'], 'Started') # Write second data frame to database. pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df2) df = pandas.read_sql('SELECT * FROM bugs', con, index_col='bug_id') self.assertEqual(len(df), 3) # Only one extra record added. self.assertEqual(df.loc[123]['status'], 'Fixed') # Bug is now fixed. self.assertItemsEqual(df.index, (123, 456, 789)) finally: con.close()
def FetchTimeseriesData(args): def _MatchesAllFilters(test_path): return all(f in test_path for f in args.filters) api = dashboard_api.PerfDashboardCommunicator(args) con = sqlite3.connect(args.database_file) try: tables.CreateIfNeeded(con) test_paths = api.ListTestPaths(args.benchmark, sheriff=args.sheriff) if args.filters: test_paths = filter(_MatchesAllFilters, test_paths) num_found = len(test_paths) print '%d test paths found!' % num_found if args.use_cache: test_paths = list(_IterStaleTestPaths(con, test_paths)) num_skipped = num_found - len(test_paths) if num_skipped: print '(skipping %d test paths already in the database)' % num_skipped for test_path in test_paths: data = api.GetTimeseries(test_path, days=args.days) timeseries = tables.timeseries.DataFrameFromJson(data) pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries) finally: con.close()
def FetchAlertsData(args): api = dashboard_api.PerfDashboardCommunicator(args) con = sqlite3.connect(args.database_file) try: tables.CreateIfNeeded(con) alerts = tables.alerts.DataFrameFromJson( api.GetAlertData(args.benchmark, args.sheriff, args.days)) print '%d alerts found!' % len(alerts) pandas_sqlite.InsertOrReplaceRecords(con, 'alerts', alerts) bug_ids = set(alerts['bug_id'].unique()) bug_ids.discard(0) # A bug_id of 0 means untriaged. print '%d bugs found!' % len(bug_ids) if args.use_cache: known_bugs = set(b for b in bug_ids if tables.bugs.Get(con, b) is not None) if known_bugs: print '(skipping %d bugs already in the database)' % len( known_bugs) bug_ids.difference_update(known_bugs) finally: con.close() total_seconds = worker_pool.Run( 'Fetching data of %d bugs: ' % len(bug_ids), _FetchBugsWorker, args, bug_ids) print '[%.1f bugs per second]' % (len(bug_ids) / total_seconds)
def FetchAlertsData(args): api = dashboard_api.PerfDashboardCommunicator(args) with tables.DbSession(args.database_file) as con: # Get alerts. num_alerts = 0 bug_ids = set() # TODO: This loop may be slow when fetching thousands of alerts, needs a # better progress indicator. for data in api.IterAlertData(args.benchmark, args.sheriff, args.days): alerts = tables.alerts.DataFrameFromJson(data) pandas_sqlite.InsertOrReplaceRecords(con, 'alerts', alerts) num_alerts += len(alerts) bug_ids.update(alerts['bug_id'].unique()) print '%d alerts found!' % num_alerts # Get set of bugs associated with those alerts. bug_ids.discard(0) # A bug_id of 0 means untriaged. print '%d bugs found!' % len(bug_ids) # Filter out bugs already in cache. if args.use_cache: known_bugs = set(b for b in bug_ids if tables.bugs.Get(con, b) is not None) if known_bugs: print '(skipping %d bugs already in the database)' % len( known_bugs) bug_ids.difference_update(known_bugs) # Use worker pool to fetch bug data. total_seconds = worker_pool.Run( 'Fetching data of %d bugs: ' % len(bug_ids), _FetchBugsWorker, args, bug_ids) print '[%.1f bugs per second]' % (len(bug_ids) / total_seconds)
def testGetMostRecentPoint_success(self): test_path = ('ChromiumPerf/android-nexus5/loading.mobile' '/timeToFirstInteractive/PageSet/Google') data = { 'test_path': test_path, 'improvement_direction': 1, 'timeseries': [ [ 'revision', 'value', 'timestamp', 'r_commit_pos', 'r_chromium' ], [ 547397, 2300.3, '2018-04-01T14:16:32.000', '547397', 'adb123' ], [ 547398, 2750.9, '2018-04-01T18:24:04.000', '547398', 'cde456' ], [ 547423, 2342.2, '2018-04-02T02:19:00.000', '547423', 'fab789' ], ] } timeseries = tables.timeseries.DataFrameFromJson(data) with tables.DbSession(':memory:') as con: pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries) point = tables.timeseries.GetMostRecentPoint(con, test_path) self.assertEqual(point['point_id'], 547423)
def testGetTimeSeries_withSummaryMetric(self): test_path = tables.timeseries.Key(test_suite='loading.mobile', measurement='timeToFirstInteractive', bot='ChromiumPerf:android-nexus5', test_case='') data = { 'improvement_direction': 'down', 'units': 'ms', 'data': [ SamplePoint(547397, 2300.3), SamplePoint(547398, 2750.9), SamplePoint(547423, 2342.2), ] } timeseries_in = tables.timeseries.DataFrameFromJson(test_path, data) with tables.DbSession(':memory:') as con: pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries_in) timeseries_out = tables.timeseries.GetTimeSeries(con, test_path) # Both DataFrame's should be equal, except the one we get out of the db # does not have an index defined. timeseries_in = timeseries_in.reset_index() self.assertTrue(timeseries_in.equals(timeseries_out))
def testInsertOrReplaceRecords_tableNotExistsRaises(self): column_types = (('bug_id', int), ('summary', str), ('status', str)) rows = [(123, 'Some bug', 'Started'), (456, 'Another bug', 'Assigned')] df = pandas_sqlite.DataFrame(column_types, index='bug_id', rows=rows) con = sqlite3.connect(':memory:') try: with self.assertRaises(AssertionError): pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df) finally: con.close()
def FetchAlertsData(args): api = dashboard_api.PerfDashboardCommunicator(args) con = sqlite3.connect(args.database_file) try: alerts = tables.alerts.DataFrameFromJson( api.GetAlertData(args.benchmark, args.days)) print '%d alerts found!' % len(alerts) pandas_sqlite.InsertOrReplaceRecords(con, 'alerts', alerts) bug_ids = set(alerts['bug_id'].unique()) bug_ids.discard(0) # A bug_id of 0 means untriaged. print '%d bugs found!' % len(bug_ids) if args.use_cache and tables.bugs.HasTable(con): known_bugs = set( b for b in bug_ids if tables.bugs.Get(con, b) is not None) if known_bugs: print '(skipping %d bugs already in the database)' % len(known_bugs) bug_ids.difference_update(known_bugs) bugs = tables.bugs.DataFrameFromJson(api.GetBugData(bug_ids)) pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', bugs) finally: con.close()
def testInsertOrReplaceRecords_tableNotExistsRaises(self): column_types = (('bug_id', int), ('summary', str), ('status', str)) columns = tuple(c for c, _ in column_types) index = columns[0] df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'), (456, 'Another bug', 'Assigned')], columns=columns, index=index) con = sqlite3.connect(':memory:') try: with self.assertRaises(AssertionError): pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df1) finally: con.close()
def Process(test_path): try: if isinstance(test_path, tables.timeseries.Key): params = test_path.AsApiParams() params['min_timestamp'] = min_timestamp data = dashboard_service.Timeseries2(**params) else: data = dashboard_service.Timeseries(test_path, days=args.days) except KeyError: logging.info('Timeseries not found: %s', test_path) return timeseries = tables.timeseries.DataFrameFromJson(test_path, data) pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries)
def testInsertOrReplaceRecords_newTable(self): columns = ('bug_id', 'summary', 'status') df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'), (456, 'Another bug', 'Assigned')], columns=columns, index=columns[0]) conn = sqlite3.connect(':memory:') try: # Write new table to database, read back and check they are equal. pandas_sqlite.InsertOrReplaceRecords(df1, 'bugs', conn) df = pandas.read_sql('SELECT * FROM bugs', conn, index_col=columns[0]) self.assertTrue(df.equals(df1)) finally: conn.close()
def FetchTimeseriesData(args): def _MatchesAllFilters(test_path): return all(f in test_path for f in args.filters) api = dashboard_api.PerfDashboardCommunicator(args) con = sqlite3.connect(args.database_file) try: test_paths = api.ListTestPaths(args.benchmark, sheriff=args.sheriff) if args.filters: test_paths = filter(_MatchesAllFilters, test_paths) print '%d test paths found!' % len(test_paths) for test_path in test_paths: data = api.GetTimeseries(test_path, days=args.days) timeseries = tables.timeseries.DataFrameFromJson(data) pandas_sqlite.InsertOrReplaceRecords(timeseries, 'timeseries', con) finally: con.close()
def testInsertOrReplaceRecords_newTable(self): # TODO(#4442): Rewrite to fail when InsertOrReplaceRecords no longer # implicitly creates the table when it doesn't exist already. columns = ('bug_id', 'summary', 'status') df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'), (456, 'Another bug', 'Assigned')], columns=columns, index=columns[0]) con = sqlite3.connect(':memory:') try: # Write new table to database, read back and check they are equal. pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df1) df = pandas.read_sql('SELECT * FROM bugs', con, index_col=columns[0]) self.assertTrue(df.equals(df1)) finally: con.close()
def testGetTimeSeries_withSummaryMetric(self): test_path = ( 'ChromiumPerf/android-nexus5/loading.mobile/timeToFirstInteractive' ) data = { 'test_path': test_path, 'improvement_direction': 1, 'timeseries': [ [ 'revision', 'value', 'timestamp', 'r_commit_pos', 'r_chromium' ], [ 547397, 2300.3, '2018-04-01T14:16:32.000', '547397', 'adb123' ], [ 547398, 2750.9, '2018-04-01T18:24:04.000', '547398', 'cde456' ], [ 547423, 2342.2, '2018-04-02T02:19:00.000', '547423', 'fab789' ], ] } timeseries_in = tables.timeseries.DataFrameFromJson(data) with tables.DbSession(':memory:') as con: pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries_in) timeseries_out = tables.timeseries.GetTimeSeries(con, test_path) print timeseries_out # Both DataFrame's should be equal, except the one we get out of the db # does not have an index defined. timeseries_in = timeseries_in.reset_index() self.assertTrue(timeseries_in.equals(timeseries_out))
def testGetMostRecentPoint_success(self): test_path = tables.timeseries.Key(test_suite='loading.mobile', measurement='timeToFirstInteractive', bot='ChromiumPerf:android-nexus5', test_case='Wikipedia') data = { 'improvement_direction': 'down', 'units': 'ms', 'data': [ SamplePoint(547397, 2300.3), SamplePoint(547398, 2750.9), SamplePoint(547423, 2342.2), ] } timeseries = tables.timeseries.DataFrameFromJson(test_path, data) with tables.DbSession(':memory:') as con: pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries) point = tables.timeseries.GetMostRecentPoint(con, test_path) self.assertEqual(point['point_id'], 547423)
def Process(test_path): data = api.GetTimeseries(test_path, days=args.days) if data: timeseries = tables.timeseries.DataFrameFromJson(data) pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries)
def Process(bug_id): bugs = tables.bugs.DataFrameFromJson(api.GetBugData(bug_id)) pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', bugs)
def Process(bug_id): bugs = tables.bugs.DataFrameFromJson([dashboard_service.Bugs(bug_id)]) pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', bugs)