def test_performance(self):
"""performance_tests only returns metrics that are numbers"""
metrics = Metrics({'int': 1, 'float': 2.0, 'complex': complex(1, 1),
'bool': True})
self.assertDictEqual(
{'int': 1, 'float': 2.0, 'complex': complex(1, 1)},
metrics.performance_metrics())
def test_performance(self):
"""performance_tests only returns metrics that are numbers"""
metrics = Metrics({
'int': 1,
'float': 2.0,
'complex': complex(1, 1),
'bool': True
})
self.assertDictEqual({
'int': 1,
'float': 2.0,
'complex': complex(1, 1)
}, metrics.performance_metrics())
def test_flatten_metrics_exported(self):
"""Exported metrics are flattened to a dict with dot-separated keys"""
# test with metrics exported from a parser
metrics = Metrics({'rps': 1})
expected = ['rps']
self.assertListEqual(expected, metrics.names)
metrics = Metrics({'latency': {'p50': 1, 'p95': 2}})
expected = ['latency.p50', 'latency.p95']
self.assertListEqual(expected, metrics.names)
# test that values are preserved correctly
metrics = Metrics({'latency': {'p50': 1, 'p95': 2}})
expected = {'latency.p50': 1, 'latency.p95': 2}
self.assertDictEqual(expected, metrics.metrics())
def test_save(self):
"""A json file is created in the right directory with the right name
when saving a job result."""
history = History('/history')
job = Job(
{
'args': ['somearg'],
'benchmark': 'bench',
'description': 'cool description',
'hooks': [],
'metrics': ['mysupercoolmetric'],
'name': 'job name',
}, {
'path': 'true',
'parser': 'parser',
})
now = datetime.now(timezone.utc)
expected_path = os.path.join(
'/history', 'job_name',
now.strftime('%Y-%m-%dT%H:%M:%SZ') + '.json')
# make sure file doesn't already exist
self.assertFalse(self.fs.Exists(expected_path))
history.save_job_result(job, Metrics({'mysupercoolmetric': 1}), now)
# make sure it exists now
self.assertTrue(self.fs.Exists(expected_path))
def test_validate_metrics(self):
"""Metrics with keys that don't match definition raise an error"""
self.job_config['metrics'] = ['rps']
job = Job(self.job_config, self.mock_benchmark)
with self.assertRaises(AssertionError):
job.validate_metrics(Metrics({}))
with self.assertRaises(AssertionError):
job.validate_metrics(Metrics({'latency': {'p50': 1}}))
self.assertTrue(job.validate_metrics(Metrics({'rps': 1})))
self.job_config['metrics'] = {'latency': ['p50', 'p95']}
job = Job(self.job_config, self.mock_benchmark)
self.assertTrue(
job.validate_metrics(Metrics({'latency': {
'p50': 1,
'p95': 2
}})))
def test_strip_metrics(self):
"""Metrics with keys that aren't in definition are removed"""
config = defaultdict(str)
config['args'] = {}
self.job_config['metrics'] = ['rps']
job = Job(self.job_config, self.mock_benchmark)
# an empty set of metrics should stay empty
stripped = job.strip_metrics(Metrics({}))
self.assertEqual(len(stripped.metrics_list()), 0)
# only passing the desired metric should stay the same
stripped = job.strip_metrics(Metrics({'rps': 1}))
self.assertEqual(len(stripped.metrics_list()), 1)
# passing in more metrics should give just the requested ones
stripped = job.strip_metrics(Metrics({'rps': 1, 'extra': 2}))
self.assertEqual(len(stripped.metrics_list()), 1)
def test_nested_names(self):
"""Nested tolerance dicts work"""
self.job.tolerances = {'a': {'b': 1}}
# Job constructor uses Metrics#flatten for tolerances as well
self.job.tolerances = Metrics.flatten(self.job.tolerances)
self.history.load_historical_results.return_value = [
metrics({'a.b': 1})
]
current = {'a.b': 4}
anomalies = analyze(current, self.job, self.history)
expected = [('a.b', 4, 0.0, 2.0)]
self.assertCountEqual(expected, anomalies)
def test_job_suite(self):
"""JobSuite runs all jobs in the suite"""
jobs = [MagicMock() for i in range(10)]
for i, job in enumerate(jobs):
job.name = str(i)
job.safe_name = str(i)
job.metrics_config.names = ['a']
job.run.return_value = Metrics({'a': i})
suite = JobSuite({'name': 'suite', 'description': 'test'}, jobs)
suite.run()
metrics = suite.run()
expected = {str(i) + '.a': i for i in range(10)}
self.assertDictEqual(expected, metrics.metrics())
def test_flatten_metrics_exported(self):
"""Exported metrics are flattened to a dict with dot-separated keys"""
# test with metrics exported from a parser
metrics = Metrics({'rps': 1})
expected = ['rps']
self.assertListEqual(expected, metrics.names)
metrics = Metrics({'latency': {'p50': 1, 'p95': 2}})
expected = ['latency.p50', 'latency.p95']
self.assertListEqual(expected, metrics.names)
# test that values are preserved correctly
metrics = Metrics({'latency': {'p50': 1, 'p95': 2}})
expected = {'latency.p50': 1, 'latency.p95': 2}
self.assertDictEqual(expected, metrics.metrics())
def test_correctness(self):
"""correctness_tests only returns metrics that are bools"""
metrics = Metrics({'rps': 1, 'second': 2, 'correct': True})
self.assertDictEqual({'correct': True}, metrics.correctness_tests())
def test_metrics_list(self):
"""metrics_list() returns a list of key-value pairs"""
metrics = Metrics({'rps': 1, 'second': 2})
self.assertListEqual([('rps', 1), ('second', 2)],
metrics.metrics_list())
def test_getitem(self):
"""Metrics getitem works the same as a dict"""
metrics = Metrics({'rps': 1})
self.assertEqual(metrics['rps'], 1)
def test_correctness(self):
"""correctness_tests only returns metrics that are bools"""
metrics = Metrics({'rps': 1, 'second': 2, 'correct': True})
self.assertDictEqual({'correct': True}, metrics.correctness_tests())
def test_metrics_list(self):
"""metrics_list() returns a list of key-value pairs"""
metrics = Metrics({'rps': 1, 'second': 2})
self.assertListEqual([('rps', 1), ('second', 2)],
metrics.metrics_list())