def testIgnoreWrongUnits(self):
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure/a',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(id=10, parent=test.key, value=100).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure/b',
improvement_direction=anomaly.DOWN,
units='wrong')
test.put()
graph_data.Row(id=10, parent=test.key, value=100).put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:bot'],
'measurement': 'measure',
'testCases': ['a', 'b'],
},
],
'statistics': ['avg'],
}
report = report_query.ReportQuery(template, [10]).FetchSync()
self.assertEqual('units', report['rows'][0]['units'])
self.assertEqual(1, len(report['rows'][0]['data'][10]['descriptors']))
self.assertEqual(
'a', report['rows'][0]['data'][10]['descriptors'][0]['testCase'])
def _External(revisions):
template = {
'rows': [],
'statistics': ['avg'],
'url': 'http://exter.nal',
}
return report_query.ReportQuery(template, revisions)
def testEmptyTestCases(self):
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(id=10, parent=test.key, value=100).put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:bot'],
'measurement': 'measure',
'testCases': [],
},
],
'statistics': ['avg'],
}
report = report_query.ReportQuery(template, [10]).FetchSync()
self.assertIsNone(
report['rows'][0]['data'][10]['descriptors'][0]['testCase'])
stats = histogram_module.RunningStatistics.FromDict(
report['rows'][0]['data'][10]['statistics'])
self.assertEqual(100, stats.mean)
def testFallBackToSuffixedTests(self):
# Unsuffixed tests without data rows should fall back.
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_avg',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(id=10, parent=test.key, value=10).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:bot'],
'measurement': 'measure',
'testCases': [],
},
],
'statistics': ['avg'],
}
report = report_query.ReportQuery(template, [10]).FetchSync()
stats = histogram_module.RunningStatistics.FromDict(
report['rows'][0]['data'][10]['statistics'])
self.assertEqual(10, stats.mean)
def testNewStyleUnsuffixedDataRows(self):
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(error=10,
id=10,
parent=test.key,
d_std=20,
d_count=30,
d_min=40,
d_max=200,
value=100).put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:bot'],
'measurement': 'measure',
'testCases': [],
},
],
'statistics': ['avg', 'std', 'count', 'min', 'max'],
}
report = report_query.ReportQuery(template, [10]).FetchSync()
stats = histogram_module.RunningStatistics.FromDict(
report['rows'][0]['data'][10]['statistics'])
self.assertEqual(100, stats.mean)
self.assertEqual(40, stats.min)
self.assertEqual(200, stats.max)
self.assertEqual(20, stats.stddev)
self.assertEqual(30, stats.count)
def testLatestRevision(self):
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(error=2, id=10, parent=test.key, value=20).put()
graph_data.Row(error=4, id=20, parent=test.key, value=40).put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:bot'],
'measurement': 'measure',
'testCases': [],
},
],
'statistics': ['avg'],
}
report = report_query.ReportQuery(template, ['latest']).FetchSync()
stats = histogram_module.RunningStatistics.FromDict(
report['rows'][0]['data']['latest']['statistics'])
self.assertEqual(40, stats.mean)
self.assertEqual(4, stats.stddev)
def GetReport(template_id, revisions):
with timing.WallTimeLogger('GetReport'), timing.CpuTimeLogger('GetReport'):
try:
template = ndb.Key('ReportTemplate', template_id).get()
except AssertionError:
# InternalOnlyModel._post_get_hook asserts that the user can access the
# entity.
return None
result = {'editable': False}
if template:
result['owners'] = template.owners
result['editable'] = utils.GetEmail() in template.owners
result['report'] = report_query.ReportQuery(
template.template, revisions).FetchSync()
else:
for handler in ListStaticTemplates():
if handler.template.key.id() != template_id:
continue
template = handler.template
report = handler(revisions)
if isinstance(report, report_query.ReportQuery):
report = report.FetchSync()
result['report'] = report
break
if template is None:
return None
result['id'] = template.key.id()
result['name'] = template.name
result['internal'] = template.internal_only
return result
def testMultipleBots(self):
test = graph_data.TestMetadata(has_rows=True,
id='master/a/suite/measure',
improvement_direction=anomaly.DOWN,
units='units')
test.UpdateSheriff()
test.put()
graph_data.Row(error=1, id=10, parent=test.key, value=10).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/b/suite/measure',
improvement_direction=anomaly.DOWN,
units='units')
test.UpdateSheriff()
test.put()
graph_data.Row(error=2, id=10, parent=test.key, value=20).put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:a', 'master:b'],
'measurement': 'measure',
'testCases': [],
},
],
'statistics': ['avg'],
}
report = report_query.ReportQuery(template, [10, 20]).FetchSync()
stats = histogram_module.RunningStatistics.FromDict(
report['rows'][0]['data'][10]['statistics'])
self.assertEqual(15, stats.mean)
def Report(revisions):
# Template is updated on each call to the handler to make sure that we use
# an up to date set of foreground/background stories.
template = {
'rows': list(IterTemplateRows(browser, bot)),
'statistics': ['avg', 'std', 'max'],
'url': 'https://bit.ly/system-health-benchmarks'
}
return report_query.ReportQuery(template, revisions)
def testCloseRevisions(self):
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure/a',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(error=1, id=9, parent=test.key, value=10).put()
graph_data.Row(error=1, id=19, parent=test.key, value=10).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure/b',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(error=2, id=9, parent=test.key, value=20).put()
graph_data.Row(error=2, id=19, parent=test.key, value=20).put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:bot'],
'measurement': 'measure',
'testCases': ['a', 'b'],
},
],
'statistics': ['avg'],
}
report = report_query.ReportQuery(template, [10, 20]).FetchSync()
data = report['rows'][0]['data']
self.assertEqual(2, len(data[10]['descriptors']))
self.assertEqual(2, len(data[20]['descriptors']))
stats = histogram_module.RunningStatistics.FromDict(
data[10]['statistics'])
self.assertEqual(15, stats.mean)
stats = histogram_module.RunningStatistics.FromDict(
data[20]['statistics'])
self.assertEqual(15, stats.mean)
def testIgnoreSuffixedDataRowsMissingAvg(self):
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_avg/a',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(id=10, parent=test.key, value=10).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_std/a',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(id=10, parent=test.key, value=20).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_std/b',
improvement_direction=anomaly.DOWN,
units='units')
test.put()
graph_data.Row(id=10, parent=test.key, value=20).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_count/a',
improvement_direction=anomaly.DOWN,
units='count')
test.put()
graph_data.Row(id=10, parent=test.key, value=30).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_count/b',
improvement_direction=anomaly.DOWN,
units='count')
test.put()
graph_data.Row(id=10, parent=test.key, value=30).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_min/a',
improvement_direction=anomaly.DOWN,
units='count')
test.put()
graph_data.Row(id=10, parent=test.key, value=4).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_min/b',
improvement_direction=anomaly.DOWN,
units='count')
test.put()
graph_data.Row(id=10, parent=test.key, value=4).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_max/a',
improvement_direction=anomaly.DOWN,
units='count')
test.put()
graph_data.Row(id=10, parent=test.key, value=500).put()
test = graph_data.TestMetadata(has_rows=True,
id='master/bot/suite/measure_max/b',
improvement_direction=anomaly.DOWN,
units='count')
test.put()
graph_data.Row(id=10, parent=test.key, value=500).put()
template = {
'rows': [
{
'testSuites': ['suite'],
'bots': ['master:bot'],
'measurement': 'measure',
'testCases': ['a', 'b'],
},
],
'statistics': ['avg', 'std', 'count'],
}
report = report_query.ReportQuery(template, [10]).FetchSync()
data = report['rows'][0]['data']
self.assertEqual(1, len(data[10]['descriptors']))
self.assertEqual('a', data[10]['descriptors'][0]['testCase'])
stats = histogram_module.RunningStatistics.FromDict(
report['rows'][0]['data'][10]['statistics'])
self.assertEqual(10, stats.mean)
self.assertEqual(4, stats.min)
self.assertEqual(500, stats.max)
self.assertEqual(20, stats.stddev)
self.assertEqual(30, stats.count)
self.assertEqual(300, stats.sum)
