def test_remove_by_bundle_name_dry_run(self):
report = model.Report(test_id='test',
bundle=model.BundleInfo(name='foo'),
date=datetime(2000, 1, 1),
results=[
model.SuiteResult(
provider='gce',
test_outcome='PASS',
),
])
report_index = model.ReportIndex()
report_index.upsert_report(report)
report_2 = model.Report(test_id='test_2',
bundle=model.BundleInfo(name='bar'),
date=datetime(2000, 2, 2),
results=[
model.SuiteResult(
provider='aws',
test_outcome='PASS',
),
])
report_index.upsert_report(report_2)
removed_reports = report_index.remove_by_bundle_name('foo',
dry_run=True)
self.assertEqual(len(removed_reports), 1)
self.assertEqual(removed_reports[0].bundle_name, 'foo')
self.assertEqual(len(report_index.reports), 2)
self.assertEqual(report_index.reports[0].bundle_name, 'bar')
self.assertEqual(report_index.reports[1].bundle_name, 'foo')
def test_filename_html(self):
report = model.Report(
test_id='test-id',
bundle=model.BundleInfo(name='cs:my-bundle'),
)
self.assertEqual(report.filename_html,
'cs_my_bundle/test-id/report.html')
def test_providers(self):
report = model.Report(results=[
model.SuiteResult(provider='aws'),
model.SuiteResult(provider='gce'),
model.SuiteResult(provider='azure'),
])
self.assertEqual(report.providers, ['aws', 'azure', 'gce'])
def test_update_from_report(self):
report = model.Report(test_id='test_id',
bundle=model.BundleInfo(name='bundle'),
date=datetime.now(),
results=[
model.SuiteResult(provider='aws',
test_outcome='PASS'),
model.SuiteResult(provider='gce',
test_outcome='FAIL'),
])
rii = model.ReportIndexItem(
test_id='my-test',
bundle_name='my-bundle',
date=datetime(2000, 1, 1),
)
self.assertEqual(rii.test_id, 'my-test')
self.assertEqual(rii.bundle_name, 'my-bundle')
self.assertEqual(rii.date, datetime(2000, 1, 1))
self.assertIsNone(rii.results)
rii.update_from_report(report)
self.assertEqual(rii.test_id, 'my-test')
self.assertEqual(rii.bundle_name, 'my-bundle')
self.assertEqual(rii.date, datetime(2000, 1, 1))
self.assertEqual(rii.results, {'aws': 'PASS', 'gce': 'FAIL'})
def test_eq(self):
rii1 = model.ReportIndexItem(test_id='test-id1', bundle_name='bundle')
rii2 = model.ReportIndexItem(test_id='test-id2', bundle_name='bundle')
rii3 = model.ReportIndexItem(test_id='test-id1', bundle_name='bundle')
report1 = model.Report(
test_id='test-id1',
bundle=model.BundleInfo(name='bundle'),
)
report2 = model.Report(
test_id='test-id2',
bundle=model.BundleInfo(name='bundle'),
)
self.assertNotEqual(rii1, rii2)
self.assertNotEqual(rii1, report2)
self.assertEqual(rii1, rii3)
self.assertEqual(rii1, report1)
def test_upsert_result(self):
result1 = model.SuiteResult(
provider='aws',
test_outcome='PASS',
tests=[
model.TestResult(
suite='bundle',
name='charm proof',
result='FAIL',
duration=0.5,
),
model.TestResult(
suite='bundle',
name='00-setup',
result='FAIL',
duration=0.5,
),
],
)
result2 = model.SuiteResult(
provider='aws',
test_outcome='PASS',
tests=[
model.TestResult(
suite='bundle',
name='charm proof',
result='PASS',
duration=0.5,
),
],
)
result3 = model.SuiteResult(
provider='gce',
test_outcome='PASS',
tests=[
model.TestResult(
suite='bundle',
name='charm proof',
result='PASS',
duration=0.5,
),
],
)
report = model.Report(
test_id='test2',
bundle=model.BundleInfo(name='cs:my-bundle'),
)
report.upsert_result(result1)
self.assertEqual(report.results, [result1])
self.assertIsNot(report.results[0], result1)
report.upsert_result(result2)
self.assertEqual(report.results, [result2])
report.upsert_result(result3)
self.assertEqual(report.results, [result2, result3])
def test_benchmark_by_name(self):
report = model.Report()
self.assertIsNone(report.benchmark_by_name('foo'))
bm = report.benchmark_by_name('foo', create=True)
self.assertIsInstance(bm, model.Benchmark)
self.assertEqual(bm.name, 'foo')
self.assertIs(report.benchmark_by_name('foo'), bm)
self.assertIsNone(report.benchmark_by_name('bar'))
bm2 = report.benchmark_by_name('bar', create=True)
self.assertIsNot(bm2, bm)
self.assertEqual(report.benchmarks, [bm, bm2])
def test_provider_result(self):
report = model.Report()
self.assertIsNone(report.provider_result('aws'))
pr = report.provider_result('aws', create=True)
self.assertIsInstance(pr, model.SuiteResult)
self.assertEqual(pr.provider, 'aws')
self.assertIs(report.provider_result('aws'), pr)
self.assertIsNone(report.provider_result('gce'))
pr2 = report.provider_result('gce', create=True)
self.assertIsNot(pr2, pr)
self.assertEqual(report.results, [pr, pr2])
def test_find_previous_report(self):
ri = model.ReportIndex(reports=[
model.ReportIndexItem(
test_id='test4',
bundle_name='bundle1',
date=datetime(2000, 1, 3),
),
model.ReportIndexItem(
test_id='test3',
bundle_name='bundle2',
date=datetime(2000, 1, 2),
),
model.ReportIndexItem(
test_id='test2',
bundle_name='bundle1',
date=datetime(2000, 1, 2),
),
model.ReportIndexItem(
test_id='test1',
bundle_name='bundle1',
date=datetime(2000, 1, 1),
),
])
report5 = model.Report(test_id='test5',
bundle=model.BundleInfo(name='bundle3'),
date=datetime(2000, 1, 3))
report4 = model.Report(test_id='test4',
bundle=model.BundleInfo(name='bundle1'),
date=datetime(2000, 1, 3))
report3 = model.Report(test_id='test3',
bundle=model.BundleInfo(name='bundle2'),
date=datetime(2000, 1, 2))
self.assertIsNone(ri.find_previous_report(report5))
self.assertIsNone(ri.find_previous_report(report3))
pr = ri.find_previous_report(report4)
self.assertEqual(pr.test_id, 'test2')
self.assertEqual(pr.bundle_name, 'bundle1')
self.assertEqual(pr.date, datetime(2000, 1, 2))
def test_upsert_report(self):
ri = model.ReportIndex()
self.assertEqual(ri.providers, [])
self.assertEqual(len(ri.reports), 0)
report = model.Report(test_id='test',
bundle=model.BundleInfo(name='bundle'),
date=datetime(2000, 1, 1),
results=[
model.SuiteResult(
provider='gce',
test_outcome='PASS',
),
])
ri.upsert_report(report)
self.assertEqual(ri.providers, ['gce'])
self.assertEqual(len(ri.reports), 1)
report.results.append(
model.SuiteResult(
provider='aws',
test_outcome='FAIL',
))
# we're not just storing a copy of the Report instance
self.assertEqual(ri.providers, ['gce'])
self.assertEqual(len(ri.reports), 1)
ri.upsert_report(report)
self.assertEqual(ri.providers, ['aws', 'gce'])
self.assertEqual(len(ri.reports), 1)
ri.upsert_report(
model.Report(test_id='test2',
bundle=model.BundleInfo(name='bundle'),
date=datetime(2000, 1, 2)))
self.assertEqual(len(ri.reports), 2)
self.assertEqual(ri.reports[0].test_id, 'test2')
def test_from_report(self):
report = model.Report(test_id='test_id',
bundle=model.BundleInfo(name='bundle'),
date=datetime.now(),
results=[
model.SuiteResult(provider='aws',
test_outcome='PASS'),
model.SuiteResult(provider='gce',
test_outcome='FAIL'),
])
rii = model.ReportIndexItem.from_report(report)
self.assertEqual(rii.test_id, 'test_id')
self.assertEqual(rii.bundle_name, 'bundle')
self.assertEqual(rii.date, report.date)
self.assertEqual(rii.results, {'aws': 'PASS', 'gce': 'FAIL'})
def load_report(self, datastore, index, test_plan):
filename = test_plan.report_filename(self.test_id)
old_filename = test_plan.old_report_filename(self.test_id)
if datastore.exists(filename):
report = model.Report.from_json(datastore.read(filename))
elif datastore.exists(old_filename):
report = model.Report.from_json(datastore.read(old_filename))
else:
report = model.Report(
version=2,
test_id=self.test_id,
date=datetime.now(),
bundle=model.BundleInfo(ref=test_plan.bundle,
name=test_plan.bundle_name,
url=test_plan.url),
)
prev_report = index.find_previous_report(report)
if prev_report:
prev_file = prev_report.filename_json
prev_report_json = datastore.read(prev_file)
prev_report = model.Report.from_json(prev_report_json)
report.upsert_benchmarks(prev_report.benchmarks)
return report
def test_upsert_benchmarks(self):
bm1 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
],
),
],
)
bm2 = model.Benchmark(
name='a2',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
],
),
],
)
bm3 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='gce',
value=1.0,
),
],
),
],
)
bm4 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
model.BenchmarkProviderResult(
provider='gce',
value=1.0,
),
],
),
],
)
bm5 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.0,
),
],
),
model.BenchmarkResult(
test_id='t2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=3.0,
),
],
),
],
)
bm6 = model.Benchmark(
name='a1',
results=[
model.BenchmarkResult(
test_id='t1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.0,
),
model.BenchmarkProviderResult(
provider='gce',
value=1.0,
),
],
),
model.BenchmarkResult(
test_id='t2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=3.0,
),
],
),
],
)
report = model.Report(
test_id='test2',
bundle=model.BundleInfo(name='cs:my-bundle'),
)
report.upsert_benchmarks(bm1)
self.assertEqual(report.benchmarks, [bm1])
self.assertIsNot(report.benchmarks[0], bm1)
report.upsert_benchmarks([bm2, bm3])
self.assertEqual(report.benchmarks, [bm4, bm2])
report.upsert_benchmarks(bm5)
self.assertEqual(report.benchmarks, [bm6, bm2])
def test_as_html_xml(self):
test_dir = os.path.dirname(__file__)
with open(os.path.join(test_dir, 'data/hadoop-processing.svg')) as fp:
svg_data = fp.read()
report = model.Report(
test_id='test2',
bundle=model.BundleInfo(name='cs:my-bundle'),
results=[
model.SuiteResult(
provider='aws',
test_outcome='PASS',
tests=[
model.TestResult(suite='bundle',
name='charm proof',
result='PASS',
duration=0.5,
output="Some output"),
model.TestResult(suite='bundle',
name='00-setup',
result='PASS',
duration=0.5,
output="Some other output"),
model.TestResult(suite='mysql',
name='00-setup',
result='PASS',
duration=1.5,
output="Some more output"),
],
),
model.SuiteResult(
provider='gce',
test_outcome='Some Failed',
tests=[
model.TestResult(
suite='bundle',
name='charm proof',
result='PASS',
duration=0.5,
),
model.TestResult(
suite='bundle',
name='00-setup',
result='PASS',
duration=0.5,
),
model.TestResult(
suite='mysql',
name='00-setup',
result='FAIL',
duration=2.5,
),
],
),
],
benchmarks=[
model.Benchmark(name='bench1',
results=[
model.BenchmarkResult(
test_id='test1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.1,
),
model.BenchmarkProviderResult(
provider='gce',
value=0.5,
),
]),
model.BenchmarkResult(
test_id='test2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.2,
),
]),
model.BenchmarkResult(
test_id='test3',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.2,
),
model.BenchmarkProviderResult(
provider='gce',
value=0.9,
),
]),
model.BenchmarkResult(
test_id='test4',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=1.0,
),
model.BenchmarkProviderResult(
provider='gce',
value=1.2,
),
]),
]),
model.Benchmark(name='bench2',
results=[
model.BenchmarkResult(
test_id='test1',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.1,
),
]),
model.BenchmarkResult(
test_id='test2',
provider_results=[
model.BenchmarkProviderResult(
provider='aws',
value=2.2,
),
]),
]),
])
html = report.as_html(None)
soup = BeautifulSoup(html, 'html.parser')
self.assertIn('Image not available', html)
self.assertEqual(soup.title.text, 'cs:my-bundle')
self.assertIn("display_chart(1, {", html)
self.assertIn("display_chart(2, {", html)
html = report.as_html(svg_data)
self.assertNotIn('Image not available', html)
self.assertIn('src="data:image/svg+xml;base64,', html)
xml = report.as_xml()
self.assertIn('Some other output', xml)
self.assertIn('testsuite', xml)