def testInsertOrReplaceRecords_existingTable(self):
columns = ('bug_id', 'summary', 'status')
df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'),
(456, 'Another bug', 'Assigned')],
columns=columns,
index=columns[0])
df2 = pandas.DataFrame.from_records([(123, 'Some bug', 'Fixed'),
(789, 'A new bug', 'Untriaged')],
columns=columns,
index=columns[0])
conn = sqlite3.connect(':memory:')
try:
# Write first data frame to database.
pandas_sqlite.InsertOrReplaceRecords(df1, 'bugs', conn)
df = pandas.read_sql('SELECT * FROM bugs',
conn,
index_col=columns[0])
self.assertEqual(len(df), 2)
self.assertEqual(df.loc[123]['status'], 'Started')
# Write second data frame to database.
pandas_sqlite.InsertOrReplaceRecords(df2, 'bugs', conn)
df = pandas.read_sql('SELECT * FROM bugs',
conn,
index_col=columns[0])
self.assertEqual(len(df), 3) # Only one extra record added.
self.assertEqual(df.loc[123]['status'],
'Fixed') # Bug is now fixed.
self.assertItemsEqual(df.index, (123, 456, 789))
finally:
conn.close()
def testInsertOrReplaceRecords_existingRecords(self):
column_types = (('bug_id', int), ('summary', str), ('status', str))
rows1 = [(123, 'Some bug', 'Started'),
(456, 'Another bug', 'Assigned')]
df1 = pandas_sqlite.DataFrame(column_types, index='bug_id', rows=rows1)
rows2 = [(123, 'Some bug', 'Fixed'), (789, 'A new bug', 'Untriaged')]
df2 = pandas_sqlite.DataFrame(column_types, index='bug_id', rows=rows2)
con = sqlite3.connect(':memory:')
try:
pandas_sqlite.CreateTableIfNotExists(con, 'bugs', df1)
# Write first data frame to database.
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df1)
df = pandas.read_sql('SELECT * FROM bugs', con, index_col='bug_id')
self.assertEqual(len(df), 2)
self.assertEqual(df.loc[123]['status'], 'Started')
# Write second data frame to database.
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df2)
df = pandas.read_sql('SELECT * FROM bugs', con, index_col='bug_id')
self.assertEqual(len(df), 3) # Only one extra record added.
self.assertEqual(df.loc[123]['status'],
'Fixed') # Bug is now fixed.
self.assertItemsEqual(df.index, (123, 456, 789))
finally:
con.close()
def FetchTimeseriesData(args):
def _MatchesAllFilters(test_path):
return all(f in test_path for f in args.filters)
api = dashboard_api.PerfDashboardCommunicator(args)
con = sqlite3.connect(args.database_file)
try:
tables.CreateIfNeeded(con)
test_paths = api.ListTestPaths(args.benchmark, sheriff=args.sheriff)
if args.filters:
test_paths = filter(_MatchesAllFilters, test_paths)
num_found = len(test_paths)
print '%d test paths found!' % num_found
if args.use_cache:
test_paths = list(_IterStaleTestPaths(con, test_paths))
num_skipped = num_found - len(test_paths)
if num_skipped:
print '(skipping %d test paths already in the database)' % num_skipped
for test_path in test_paths:
data = api.GetTimeseries(test_path, days=args.days)
timeseries = tables.timeseries.DataFrameFromJson(data)
pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries)
finally:
con.close()
def FetchAlertsData(args):
api = dashboard_api.PerfDashboardCommunicator(args)
con = sqlite3.connect(args.database_file)
try:
tables.CreateIfNeeded(con)
alerts = tables.alerts.DataFrameFromJson(
api.GetAlertData(args.benchmark, args.sheriff, args.days))
print '%d alerts found!' % len(alerts)
pandas_sqlite.InsertOrReplaceRecords(con, 'alerts', alerts)
bug_ids = set(alerts['bug_id'].unique())
bug_ids.discard(0) # A bug_id of 0 means untriaged.
print '%d bugs found!' % len(bug_ids)
if args.use_cache:
known_bugs = set(b for b in bug_ids
if tables.bugs.Get(con, b) is not None)
if known_bugs:
print '(skipping %d bugs already in the database)' % len(
known_bugs)
bug_ids.difference_update(known_bugs)
finally:
con.close()
total_seconds = worker_pool.Run(
'Fetching data of %d bugs: ' % len(bug_ids), _FetchBugsWorker, args,
bug_ids)
print '[%.1f bugs per second]' % (len(bug_ids) / total_seconds)
def FetchAlertsData(args):
api = dashboard_api.PerfDashboardCommunicator(args)
with tables.DbSession(args.database_file) as con:
# Get alerts.
num_alerts = 0
bug_ids = set()
# TODO: This loop may be slow when fetching thousands of alerts, needs a
# better progress indicator.
for data in api.IterAlertData(args.benchmark, args.sheriff, args.days):
alerts = tables.alerts.DataFrameFromJson(data)
pandas_sqlite.InsertOrReplaceRecords(con, 'alerts', alerts)
num_alerts += len(alerts)
bug_ids.update(alerts['bug_id'].unique())
print '%d alerts found!' % num_alerts
# Get set of bugs associated with those alerts.
bug_ids.discard(0) # A bug_id of 0 means untriaged.
print '%d bugs found!' % len(bug_ids)
# Filter out bugs already in cache.
if args.use_cache:
known_bugs = set(b for b in bug_ids
if tables.bugs.Get(con, b) is not None)
if known_bugs:
print '(skipping %d bugs already in the database)' % len(
known_bugs)
bug_ids.difference_update(known_bugs)
# Use worker pool to fetch bug data.
total_seconds = worker_pool.Run(
'Fetching data of %d bugs: ' % len(bug_ids), _FetchBugsWorker, args,
bug_ids)
print '[%.1f bugs per second]' % (len(bug_ids) / total_seconds)
def testGetMostRecentPoint_success(self):
test_path = ('ChromiumPerf/android-nexus5/loading.mobile'
'/timeToFirstInteractive/PageSet/Google')
data = {
'test_path':
test_path,
'improvement_direction':
1,
'timeseries': [
[
'revision', 'value', 'timestamp', 'r_commit_pos',
'r_chromium'
],
[
547397, 2300.3, '2018-04-01T14:16:32.000', '547397',
'adb123'
],
[
547398, 2750.9, '2018-04-01T18:24:04.000', '547398',
'cde456'
],
[
547423, 2342.2, '2018-04-02T02:19:00.000', '547423',
'fab789'
],
]
}
timeseries = tables.timeseries.DataFrameFromJson(data)
with tables.DbSession(':memory:') as con:
pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries)
point = tables.timeseries.GetMostRecentPoint(con, test_path)
self.assertEqual(point['point_id'], 547423)
def testGetTimeSeries_withSummaryMetric(self):
test_path = tables.timeseries.Key(test_suite='loading.mobile',
measurement='timeToFirstInteractive',
bot='ChromiumPerf:android-nexus5',
test_case='')
data = {
'improvement_direction':
'down',
'units':
'ms',
'data': [
SamplePoint(547397, 2300.3),
SamplePoint(547398, 2750.9),
SamplePoint(547423, 2342.2),
]
}
timeseries_in = tables.timeseries.DataFrameFromJson(test_path, data)
with tables.DbSession(':memory:') as con:
pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries',
timeseries_in)
timeseries_out = tables.timeseries.GetTimeSeries(con, test_path)
# Both DataFrame's should be equal, except the one we get out of the db
# does not have an index defined.
timeseries_in = timeseries_in.reset_index()
self.assertTrue(timeseries_in.equals(timeseries_out))
def testInsertOrReplaceRecords_tableNotExistsRaises(self):
column_types = (('bug_id', int), ('summary', str), ('status', str))
rows = [(123, 'Some bug', 'Started'), (456, 'Another bug', 'Assigned')]
df = pandas_sqlite.DataFrame(column_types, index='bug_id', rows=rows)
con = sqlite3.connect(':memory:')
try:
with self.assertRaises(AssertionError):
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df)
finally:
con.close()
def FetchAlertsData(args):
api = dashboard_api.PerfDashboardCommunicator(args)
con = sqlite3.connect(args.database_file)
try:
alerts = tables.alerts.DataFrameFromJson(
api.GetAlertData(args.benchmark, args.days))
print '%d alerts found!' % len(alerts)
pandas_sqlite.InsertOrReplaceRecords(con, 'alerts', alerts)
bug_ids = set(alerts['bug_id'].unique())
bug_ids.discard(0) # A bug_id of 0 means untriaged.
print '%d bugs found!' % len(bug_ids)
if args.use_cache and tables.bugs.HasTable(con):
known_bugs = set(
b for b in bug_ids if tables.bugs.Get(con, b) is not None)
if known_bugs:
print '(skipping %d bugs already in the database)' % len(known_bugs)
bug_ids.difference_update(known_bugs)
bugs = tables.bugs.DataFrameFromJson(api.GetBugData(bug_ids))
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', bugs)
finally:
con.close()
def testInsertOrReplaceRecords_tableNotExistsRaises(self):
column_types = (('bug_id', int), ('summary', str), ('status', str))
columns = tuple(c for c, _ in column_types)
index = columns[0]
df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'),
(456, 'Another bug', 'Assigned')],
columns=columns,
index=index)
con = sqlite3.connect(':memory:')
try:
with self.assertRaises(AssertionError):
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df1)
finally:
con.close()
def Process(test_path):
try:
if isinstance(test_path, tables.timeseries.Key):
params = test_path.AsApiParams()
params['min_timestamp'] = min_timestamp
data = dashboard_service.Timeseries2(**params)
else:
data = dashboard_service.Timeseries(test_path, days=args.days)
except KeyError:
logging.info('Timeseries not found: %s', test_path)
return
timeseries = tables.timeseries.DataFrameFromJson(test_path, data)
pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries)
def testInsertOrReplaceRecords_newTable(self):
columns = ('bug_id', 'summary', 'status')
df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'),
(456, 'Another bug', 'Assigned')],
columns=columns,
index=columns[0])
conn = sqlite3.connect(':memory:')
try:
# Write new table to database, read back and check they are equal.
pandas_sqlite.InsertOrReplaceRecords(df1, 'bugs', conn)
df = pandas.read_sql('SELECT * FROM bugs',
conn,
index_col=columns[0])
self.assertTrue(df.equals(df1))
finally:
conn.close()
def FetchTimeseriesData(args):
def _MatchesAllFilters(test_path):
return all(f in test_path for f in args.filters)
api = dashboard_api.PerfDashboardCommunicator(args)
con = sqlite3.connect(args.database_file)
try:
test_paths = api.ListTestPaths(args.benchmark, sheriff=args.sheriff)
if args.filters:
test_paths = filter(_MatchesAllFilters, test_paths)
print '%d test paths found!' % len(test_paths)
for test_path in test_paths:
data = api.GetTimeseries(test_path, days=args.days)
timeseries = tables.timeseries.DataFrameFromJson(data)
pandas_sqlite.InsertOrReplaceRecords(timeseries, 'timeseries', con)
finally:
con.close()
def testInsertOrReplaceRecords_newTable(self):
# TODO(#4442): Rewrite to fail when InsertOrReplaceRecords no longer
# implicitly creates the table when it doesn't exist already.
columns = ('bug_id', 'summary', 'status')
df1 = pandas.DataFrame.from_records([(123, 'Some bug', 'Started'),
(456, 'Another bug', 'Assigned')],
columns=columns,
index=columns[0])
con = sqlite3.connect(':memory:')
try:
# Write new table to database, read back and check they are equal.
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', df1)
df = pandas.read_sql('SELECT * FROM bugs',
con,
index_col=columns[0])
self.assertTrue(df.equals(df1))
finally:
con.close()
def testGetTimeSeries_withSummaryMetric(self):
test_path = (
'ChromiumPerf/android-nexus5/loading.mobile/timeToFirstInteractive'
)
data = {
'test_path':
test_path,
'improvement_direction':
1,
'timeseries': [
[
'revision', 'value', 'timestamp', 'r_commit_pos',
'r_chromium'
],
[
547397, 2300.3, '2018-04-01T14:16:32.000', '547397',
'adb123'
],
[
547398, 2750.9, '2018-04-01T18:24:04.000', '547398',
'cde456'
],
[
547423, 2342.2, '2018-04-02T02:19:00.000', '547423',
'fab789'
],
]
}
timeseries_in = tables.timeseries.DataFrameFromJson(data)
with tables.DbSession(':memory:') as con:
pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries',
timeseries_in)
timeseries_out = tables.timeseries.GetTimeSeries(con, test_path)
print timeseries_out
# Both DataFrame's should be equal, except the one we get out of the db
# does not have an index defined.
timeseries_in = timeseries_in.reset_index()
self.assertTrue(timeseries_in.equals(timeseries_out))
def testGetMostRecentPoint_success(self):
test_path = tables.timeseries.Key(test_suite='loading.mobile',
measurement='timeToFirstInteractive',
bot='ChromiumPerf:android-nexus5',
test_case='Wikipedia')
data = {
'improvement_direction':
'down',
'units':
'ms',
'data': [
SamplePoint(547397, 2300.3),
SamplePoint(547398, 2750.9),
SamplePoint(547423, 2342.2),
]
}
timeseries = tables.timeseries.DataFrameFromJson(test_path, data)
with tables.DbSession(':memory:') as con:
pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries)
point = tables.timeseries.GetMostRecentPoint(con, test_path)
self.assertEqual(point['point_id'], 547423)
def Process(test_path):
data = api.GetTimeseries(test_path, days=args.days)
if data:
timeseries = tables.timeseries.DataFrameFromJson(data)
pandas_sqlite.InsertOrReplaceRecords(con, 'timeseries', timeseries)
def Process(bug_id):
bugs = tables.bugs.DataFrameFromJson(api.GetBugData(bug_id))
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', bugs)
def Process(bug_id):
bugs = tables.bugs.DataFrameFromJson([dashboard_service.Bugs(bug_id)])
pandas_sqlite.InsertOrReplaceRecords(con, 'bugs', bugs)
