def test_lint_results(self):
with self.assertRaisesRegexp(TypeError, r'"SomeTest" does not contain metrics or tests'):
BenchmarkResults._lint_results({'SomeTest': {}})
with self.assertRaisesRegexp(TypeError, r'The metrics in "SomeTest" is not a dictionary'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': []}})
with self.assertRaisesRegexp(TypeError, r'The aggregator list is empty in "Time" metric of "SomeTest"'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': []}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" is not wrapped by a configuration; e.g. "current"'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': [1, 2]}}})
self.assertTrue(BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': {'current': [1, 2]}}}}))
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" was not an aggregator list or a dictionary of configurations: 1'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': 1}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" contains non-numeric value: \["Total"\]'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': {'current': ['Total']}}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" contains non-numeric value: \["Total", "Geometric"\]'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': {'current': [['Total', 'Geometric']]}}}})
with self.assertRaisesRegexp(TypeError, r'"SomeTest" requires aggregation but it has no subtests'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total']}}})
with self.assertRaisesRegexp(TypeError, r'"OtherTest" requires aggregation but it has no subtests'):
BenchmarkResults._lint_results({'OtherTest': {'metrics': {'Time': ['Total']}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" had invalid aggregator list: \["Total", "Total"\]'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total', 'Total']}, 'tests': {
'SubTest1': {'metrics': {'Time': {'current': []}}}}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" uses unknown aggregator: KittenMean'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['KittenMean']}, 'tests': {
'SubTest1': {'metrics': {'Time': {'current': []}}}}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" had a mismatching subtest values'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total']}, 'tests': {
'SubTest1': {'metrics': {'Time': {'current': [1, 2, 3]}}},
'SubTest2': {'metrics': {'Time': {'current': [4, 5, 6, 7]}}}}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" had a mismatching subtest values'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total']}, 'tests': {
'SubTest1': {'metrics': {'Time': {'current': [[1, 2], [3]]}}},
'SubTest2': {'metrics': {'Time': {'current': [[4, 5], [6, 7]]}}}}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" had malformed values: \[1, \[2\], 3\]'):
BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': {'current': [1, [2], 3]}}}})
with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" has no value to aggregate as "Arithmetic" in a subtest "SubTest1"'):
BenchmarkResults._lint_results({'SomeTest': {
'metrics': {'Time': ['Arithmetic']},
'tests': {
'SubTest1': {
'metrics': {'Time': ['Total', 'Geometric']},
'tests': {
'GrandChild1': {'metrics': {'Time': {'current': [1, 2]}}},
'GrandChild2': {'metrics': {'Time': {'current': [3, 4]}}}}},
'SubTest2': {'metrics': {'Time': {'current': [9, 24]}}}}}})
self.assertTrue(BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total']}, 'tests': {
'SubTest1': {'metrics': {'Time': {'current': [1, 2, 3]}}},
'SubTest2': {'metrics': {'Time': {'current': [4, 5, 6], 'baseline': [7]}}}}}}))
self.assertTrue(BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total']}, 'tests': {
'SubTest1': {'metrics': {'Time': {'current': [1, 2, 3]}}},
'SubTest2': {'metrics': {'Runs': {'current': [4, 5, 6, 7]}}}}}}))
self.assertTrue(BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total']}, 'tests': {
'SubTest1': {'metrics': {'Time': {'current': [[1, 2], [3, 4]]}}},
'SubTest2': {'metrics': {'Time': {'current': [[5, 6], [7, 8]]}}}}}}))
