def test_result_by_test_id(self):
bm = model.Benchmark(results=[
model.BenchmarkResult(test_id='test1'),
model.BenchmarkResult(test_id='test2'),
])
self.assertEqual(bm.result_by_test_id('test1').test_id, 'test1')
self.assertEqual(bm.result_by_test_id('test2').test_id, 'test2')
self.assertIsNone(bm.result_by_test_id('test3'))
self.assertEqual(
bm.result_by_test_id('test3', create=True).test_id, 'test3')
def test_providers(self):
bm = model.Benchmark(results=[
model.BenchmarkResult(provider_results=[
model.BenchmarkProviderResult(provider='aws'),
model.BenchmarkProviderResult(provider='gce'),
]),
model.BenchmarkResult(provider_results=[
model.BenchmarkProviderResult(provider='gce'),
model.BenchmarkProviderResult(provider='azure'),
]),
])
self.assertEqual(bm.providers, ['aws', 'azure', 'gce'])
def test_provider_value(self, provider_result):
br = model.BenchmarkResult()
provider_result.return_value = None
self.assertIs(br.provider_value('foo'), None)
provider_result.return_value = mock.Mock(value='test-value')
self.assertEqual(br.provider_value('foo'), 'test-value')
def test_provider_result(self):
br = model.BenchmarkResult(provider_results=[
model.BenchmarkProviderResult(provider='aws'),
model.BenchmarkProviderResult(provider='gce'),
])
self.assertEqual(br.provider_result('aws').provider, 'aws')
self.assertEqual(br.provider_result('gce').provider, 'gce')
self.assertIsNone(br.provider_result('azure'))
def test_upsert_benchmarks(self):
bm1 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
],
),
],
)
bm2 = model.Benchmark(
name='a2',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
],
),
],
)
bm3 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='gce',
value=1.0,
),
],
),
],
)
bm4 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
model.BenchmarkProviderResult(
provider='gce',
value=1.0,
),
],
),
],
)
bm5 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.0,
),
],
),
model.BenchmarkResult(
test_id='t2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=3.0,
),
],
),
],
)
bm6 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.0,
),
model.BenchmarkProviderResult(
provider='gce',
value=1.0,
),
],
),
model.BenchmarkResult(
test_id='t2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=3.0,
),
],
),
],
)
report = model.Report(
test_id='test2',
bundle=model.BundleInfo(name='cs:my-bundle'),
)
report.upsert_benchmarks(bm1)
self.assertEqual(report.benchmarks, [bm1])
self.assertIsNot(report.benchmarks[0], bm1)
report.upsert_benchmarks([bm2, bm3])
self.assertEqual(report.benchmarks, [bm4, bm2])
report.upsert_benchmarks(bm5)
self.assertEqual(report.benchmarks, [bm6, bm2])
def test_as_html_xml(self):
test_dir = os.path.dirname(__file__)
with open(os.path.join(test_dir, 'data/hadoop-processing.svg')) as fp:
svg_data = fp.read()
report = model.Report(
test_id='test2',
bundle=model.BundleInfo(name='cs:my-bundle'),
results=[
model.SuiteResult(
provider='aws',
test_outcome='PASS',
tests=[
model.TestResult(suite='bundle',
name='charm proof',
result='PASS',
duration=0.5,
output="Some output"),
model.TestResult(suite='bundle',
name='00-setup',
result='PASS',
duration=0.5,
output="Some other output"),
model.TestResult(suite='mysql',
name='00-setup',
result='PASS',
duration=1.5,
output="Some more output"),
],
),
model.SuiteResult(
provider='gce',
test_outcome='Some Failed',
tests=[
model.TestResult(
suite='bundle',
name='charm proof',
result='PASS',
duration=0.5,
),
model.TestResult(
suite='bundle',
name='00-setup',
result='PASS',
duration=0.5,
),
model.TestResult(
suite='mysql',
name='00-setup',
result='FAIL',
duration=2.5,
),
],
),
],
benchmarks=[
model.Benchmark(name='bench1',
results=[
model.BenchmarkResult(
test_id='test1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.1,
),
model.BenchmarkProviderResult(
provider='gce',
value=0.5,
),
]),
model.BenchmarkResult(
test_id='test2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.2,
),
]),
model.BenchmarkResult(
test_id='test3',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.2,
),
model.BenchmarkProviderResult(
provider='gce',
value=0.9,
),
]),
model.BenchmarkResult(
test_id='test4',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
model.BenchmarkProviderResult(
provider='gce',
value=1.2,
),
]),
]),
model.Benchmark(name='bench2',
results=[
model.BenchmarkResult(
test_id='test1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.1,
),
]),
model.BenchmarkResult(
test_id='test2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.2,
),
]),
]),
])
html = report.as_html(None)
soup = BeautifulSoup(html, 'html.parser')
self.assertIn('Image not available', html)
self.assertEqual(soup.title.text, 'cs:my-bundle')
self.assertIn("display_chart(1, {", html)
self.assertIn("display_chart(2, {", html)
html = report.as_html(svg_data)
self.assertNotIn('Image not available', html)
self.assertIn('src="data:image/svg+xml;base64,', html)
xml = report.as_xml()
self.assertIn('Some other output', xml)
self.assertIn('testsuite', xml)
def test_as_chart(self):
bm = model.Benchmark(
name='terasort',
results=[
model.BenchmarkResult(
test_id='test_1',
provider_results=[
model.BenchmarkProviderResult(provider='aws',
value=2.0),
model.BenchmarkProviderResult(provider='gce',
value=1.5),
]),
model.BenchmarkResult(
test_id='test_2',
provider_results=[
model.BenchmarkProviderResult(provider='gce',
value=0.5),
model.BenchmarkProviderResult(provider='azure',
value=2.0),
]),
])
self.assertEqual(
bm.as_chart(), {
'title':
'terasort',
'labels': ['test_1', 'test_2'],
'min':
0.5,
'max':
2.0,
'datasets': [
{
'label': 'aws',
'fill': False,
'borderColor': '#0000ee',
'borderWidth': 2,
'backgroundColor': '#0000ee',
'lineTension': 0,
'spanGaps': True,
'data': [
2.0,
None,
],
},
{
'label': 'azure',
'fill': False,
'borderColor': '#ff00ff',
'borderWidth': 2,
'backgroundColor': '#ff00ff',
'lineTension': 0,
'spanGaps': True,
'data': [
None,
2.0,
],
},
{
'label': 'gce',
'fill': False,
'borderColor': '#009900',
'borderWidth': 2,
'backgroundColor': '#009900',
'lineTension': 0,
'spanGaps': True,
'data': [
1.5,
0.5,
],
},
]
})