def test_single_table_report(self):
"""Single table report has inline headers and no elaborate sections."""
self.tc.removed = [] # test handling empty section
rf = ReportFormatter(self.tc, changes_only=True, single_table=True)
markdown = rf.markdown()
self.assertNotIn('<details', markdown) # no sections
self.assertNotIn('\n\n', markdown) # table must not be broken
self.assertNotIn('Removed', markdown)
self.assert_report_contains([
'\n**Regression** ', '| **OLD**', '| **NEW**', '| **DELTA**',
'| **RATIO**', '\n**Added** ', '| **MIN**', '| **MAX**',
'| **MEAN**', '| **MAX_RSS**'
], markdown)
# Single delimiter row:
self.assertIn('\n:---', markdown) # first column is left aligned
self.assertEqual(markdown.count('| ---:'), 4) # other, right aligned
# Separator before every inline header (new section):
self.assertEqual(markdown.count(' | | | | '), 2)
git = rf.git()
self.assertNotIn('): \n', git) # no sections
self.assertNotIn('REMOVED', git)
self.assert_report_contains([
'\nREGRESSION ', ' OLD ', ' NEW ', ' DELTA ', ' RATIO ',
'\n\nADDED ', ' MIN ', ' MAX ', ' MEAN ', ' MAX_RSS '
], git)
# Separator before every inline header (new section):
self.assertEqual(git.count('\n\n'), 2)
def test_report_only_changes(self):
"""Leave out tests without significant change."""
rf = ReportFormatter(self.tc, changes_only=True)
markdown, git, html = rf.markdown(), rf.git(), rf.html()
self.assertNotIn("No Changes", markdown)
self.assertNotIn("AngryPhonebook", markdown)
self.assertNotIn("No Changes", git)
self.assertNotIn("AngryPhonebook", git)
self.assertNotIn("No Changes", html)
self.assertNotIn("AngryPhonebook", html)
def test_report_only_changes(self):
"""Leave out tests without significant change."""
rf = ReportFormatter(self.tc, '', '', changes_only=True)
markdown, git, html = rf.markdown(), rf.git(), rf.html()
self.assertNotIn('No Changes', markdown)
self.assertNotIn('AngryPhonebook', markdown)
self.assertNotIn('No Changes', git)
self.assertNotIn('AngryPhonebook', git)
self.assertNotIn('No Changes', html)
self.assertNotIn('AngryPhonebook', html)
def test_single_table_report(self):
"""Single table report has inline headers and no elaborate sections."""
self.tc.removed = [] # test handling empty section
rf = ReportFormatter(self.tc, changes_only=True, single_table=True)
markdown = rf.markdown()
self.assertNotIn("<details", markdown) # no sections
self.assertNotIn("\n\n", markdown) # table must not be broken
self.assertNotIn("Removed", markdown)
self.assert_report_contains(
[
"\n**Regression** ",
"| **OLD**",
"| **NEW**",
"| **DELTA**",
"| **RATIO**",
"\n**Added** ",
"| **MIN**",
"| **MAX**",
"| **MEAN**",
"| **MAX_RSS**",
],
markdown,
)
# Single delimiter row:
self.assertIn("\n:---", markdown) # first column is left aligned
self.assertEqual(markdown.count("| ---:"), 4) # other, right aligned
# Separator before every inline header (new section):
self.assertEqual(markdown.count(" | | | | "), 2)
git = rf.git()
self.assertNotIn("): \n", git) # no sections
self.assertNotIn("REMOVED", git)
self.assert_report_contains(
[
"\nREGRESSION ",
" OLD ",
" NEW ",
" DELTA ",
" RATIO ",
"\n\nADDED ",
" MIN ",
" MAX ",
" MEAN ",
" MAX_RSS ",
],
git,
)
# Separator before every inline header (new section):
self.assertEqual(git.count("\n\n"), 2)
class TestReportFormatter(OldAndNewLog):
def setUp(self):
super(TestReportFormatter, self).setUp()
self.tc = TestComparator(self.old_results, self.new_results, 0.05)
self.rf = ReportFormatter(self.tc, changes_only=False)
self.markdown = self.rf.markdown()
self.git = self.rf.git()
self.html = self.rf.html()
def assert_markdown_contains(self, texts):
self.assert_report_contains(texts, self.markdown)
def assert_git_contains(self, texts):
self.assert_report_contains(texts, self.git)
def assert_html_contains(self, texts):
self.assert_report_contains(texts, self.html)
def test_values(self):
self.assertEqual(
ReportFormatter.values(
PerformanceTestResult(
"1,AngryPhonebook,20,10664,12933,11035,576,10884".split(
","))),
("AngryPhonebook", "10664", "12933", "11035", "—"),
)
self.assertEqual(
ReportFormatter.values(
PerformanceTestResult(
"1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336".
split(","))),
("AngryPhonebook", "12045", "12045", "12045", "10510336"),
)
r1 = PerformanceTestResult(
"1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336".split(","))
r2 = PerformanceTestResult(
"1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144".split(","))
self.assertEqual(
ReportFormatter.values(ResultComparison(r1, r2)),
("AngryPhonebook", "12325", "11616", "-5.8%", "1.06x"),
)
self.assertEqual(
ReportFormatter.values(ResultComparison(r2, r1)),
("AngryPhonebook", "11616", "12325", "+6.1%", "0.94x"),
)
r2.max = r1.min + 1
self.assertEqual(
ReportFormatter.values(ResultComparison(r1, r2))[4],
"1.06x (?)", # is_dubious
)
def test_justified_columns(self):
"""Table columns are all formated with same width, defined by the
longest value.
"""
self.assert_markdown_contains([
"AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445",
"Array2D | 335831 | 335831 | +0.0% | 1.00x",
])
self.assert_git_contains([
"AnyHashableWithAClass 247027 319065 259056 10250445",
"Array2D 335831 335831 +0.0% 1.00x",
])
def test_column_headers(self):
"""Report contains table headers for ResultComparisons and changed
PerformanceTestResults.
"""
performance_test_result = self.tc.added[0]
self.assertEqual(
ReportFormatter.header_for(performance_test_result),
("TEST", "MIN", "MAX", "MEAN", "MAX_RSS"),
)
comparison_result = self.tc.increased[0]
self.assertEqual(
ReportFormatter.header_for(comparison_result),
("TEST", "OLD", "NEW", "DELTA", "RATIO"),
)
self.assert_markdown_contains([
"TEST | OLD | NEW | DELTA | RATIO",
":--- | ---: | ---: | ---: | ---: ",
"TEST | MIN | MAX | MEAN | MAX_RSS",
])
self.assert_git_contains([
"TEST OLD NEW DELTA RATIO",
"TEST MIN MAX MEAN MAX_RSS",
])
self.assert_html_contains([
"""
<th align='left'>OLD</th>
<th align='left'>NEW</th>
<th align='left'>DELTA</th>
<th align='left'>RATIO</th>""",
"""
<th align='left'>MIN</th>
<th align='left'>MAX</th>
<th align='left'>MEAN</th>
<th align='left'>MAX_RSS</th>""",
])
def test_emphasize_speedup(self):
"""Emphasize speedup values for regressions and improvements"""
# tests in No Changes don't have emphasized speedup
self.assert_markdown_contains([
"BitCount | 3 | 9 | +199.9% | **0.33x**",
"ByteSwap | 4 | 0 | -100.0% | **4001.00x**",
"AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ",
"ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**",
])
self.assert_git_contains([
"BitCount 3 9 +199.9% **0.33x**",
"ByteSwap 4 0 -100.0% **4001.00x**",
"AngryPhonebook 10458 10458 +0.0% 1.00x",
"ArrayAppend 23641 20000 -15.4% **1.18x (?)**",
])
self.assert_html_contains([
"""
<tr>
<td align='left'>BitCount</td>
<td align='left'>3</td>
<td align='left'>9</td>
<td align='left'>+199.9%</td>
<td align='left'><font color='red'>0.33x</font></td>
</tr>""",
"""
<tr>
<td align='left'>ByteSwap</td>
<td align='left'>4</td>
<td align='left'>0</td>
<td align='left'>-100.0%</td>
<td align='left'><font color='green'>4001.00x</font></td>
</tr>""",
"""
<tr>
<td align='left'>AngryPhonebook</td>
<td align='left'>10458</td>
<td align='left'>10458</td>
<td align='left'>+0.0%</td>
<td align='left'><font color='black'>1.00x</font></td>
</tr>""",
])
def test_sections(self):
"""Report is divided into sections with summaries."""
self.assert_markdown_contains([
"""<details open>
<summary>Regression (1)</summary>""",
"""<details >
<summary>Improvement (2)</summary>""",
"""<details >
<summary>No Changes (2)</summary>""",
"""<details open>
<summary>Added (1)</summary>""",
"""<details open>
<summary>Removed (1)</summary>""",
])
self.assert_git_contains([
"Regression (1): \n",
"Improvement (2): \n",
"No Changes (2): \n",
"Added (1): \n",
"Removed (1): \n",
])
self.assert_html_contains([
"<th align='left'>Regression (1)</th>",
"<th align='left'>Improvement (2)</th>",
"<th align='left'>No Changes (2)</th>",
"<th align='left'>Added (1)</th>",
"<th align='left'>Removed (1)</th>",
])
def test_report_only_changes(self):
"""Leave out tests without significant change."""
rf = ReportFormatter(self.tc, changes_only=True)
markdown, git, html = rf.markdown(), rf.git(), rf.html()
self.assertNotIn("No Changes", markdown)
self.assertNotIn("AngryPhonebook", markdown)
self.assertNotIn("No Changes", git)
self.assertNotIn("AngryPhonebook", git)
self.assertNotIn("No Changes", html)
self.assertNotIn("AngryPhonebook", html)
def test_single_table_report(self):
"""Single table report has inline headers and no elaborate sections."""
self.tc.removed = [] # test handling empty section
rf = ReportFormatter(self.tc, changes_only=True, single_table=True)
markdown = rf.markdown()
self.assertNotIn("<details", markdown) # no sections
self.assertNotIn("\n\n", markdown) # table must not be broken
self.assertNotIn("Removed", markdown)
self.assert_report_contains(
[
"\n**Regression** ",
"| **OLD**",
"| **NEW**",
"| **DELTA**",
"| **RATIO**",
"\n**Added** ",
"| **MIN**",
"| **MAX**",
"| **MEAN**",
"| **MAX_RSS**",
],
markdown,
)
# Single delimiter row:
self.assertIn("\n:---", markdown) # first column is left aligned
self.assertEqual(markdown.count("| ---:"), 4) # other, right aligned
# Separator before every inline header (new section):
self.assertEqual(markdown.count(" | | | | "), 2)
git = rf.git()
self.assertNotIn("): \n", git) # no sections
self.assertNotIn("REMOVED", git)
self.assert_report_contains(
[
"\nREGRESSION ",
" OLD ",
" NEW ",
" DELTA ",
" RATIO ",
"\n\nADDED ",
" MIN ",
" MAX ",
" MEAN ",
" MAX_RSS ",
],
git,
)
# Separator before every inline header (new section):
self.assertEqual(git.count("\n\n"), 2)
class TestReportFormatter(OldAndNewLog):
def setUp(self):
super(TestReportFormatter, self).setUp()
self.tc = TestComparator(self.old_log, self.new_log, 0.05)
self.rf = ReportFormatter(self.tc, '', '', changes_only=False)
self.markdown = self.rf.markdown()
self.git = self.rf.git()
self.html = self.rf.html()
def assert_markdown_contains(self, texts):
self.assert_report_contains(texts, self.markdown)
def assert_git_contains(self, texts):
self.assert_report_contains(texts, self.git)
def assert_html_contains(self, texts):
self.assert_report_contains(texts, self.html)
def test_justified_columns(self):
"""Table columns are all formated with same width, defined by the
longest value.
"""
self.assert_markdown_contains([
'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445',
'Array2D | 335831 | 335831 | +0.0% | 1.00x'])
self.assert_git_contains([
'AnyHashableWithAClass 247027 319065 259056 10250445',
'Array2D 335831 335831 +0.0% 1.00x'])
def test_column_headers(self):
"""Report contains table headers for ResultComparisons and changed
PerformanceTestResults.
"""
self.assert_markdown_contains([
'TEST | OLD | NEW | DELTA | SPEEDUP',
'--- | --- | --- | --- | --- ',
'TEST | MIN | MAX | MEAN | MAX_RSS'])
self.assert_git_contains([
'TEST OLD NEW DELTA SPEEDUP',
'TEST MIN MAX MEAN MAX_RSS'])
self.assert_html_contains([
"""
<th align='left'>OLD</th>
<th align='left'>NEW</th>
<th align='left'>DELTA</th>
<th align='left'>SPEEDUP</th>""",
"""
<th align='left'>MIN</th>
<th align='left'>MAX</th>
<th align='left'>MEAN</th>
<th align='left'>MAX_RSS</th>"""])
def test_emphasize_speedup(self):
"""Emphasize speedup values for regressions and improvements"""
# tests in No Changes don't have emphasized speedup
self.assert_markdown_contains([
'BitCount | 3 | 9 | +199.9% | **0.33x**',
'ByteSwap | 4 | 0 | -100.0% | **4001.00x**',
'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ',
'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**'
])
self.assert_git_contains([
'BitCount 3 9 +199.9% **0.33x**',
'ByteSwap 4 0 -100.0% **4001.00x**',
'AngryPhonebook 10458 10458 +0.0% 1.00x',
'ArrayAppend 23641 20000 -15.4% **1.18x (?)**'
])
self.assert_html_contains([
"""
<tr>
<td align='left'>BitCount</td>
<td align='left'>3</td>
<td align='left'>9</td>
<td align='left'>+199.9%</td>
<td align='left'><font color='red'>0.33x</font></td>
</tr>""",
"""
<tr>
<td align='left'>ByteSwap</td>
<td align='left'>4</td>
<td align='left'>0</td>
<td align='left'>-100.0%</td>
<td align='left'><font color='green'>4001.00x</font></td>
</tr>""",
"""
<tr>
<td align='left'>AngryPhonebook</td>
<td align='left'>10458</td>
<td align='left'>10458</td>
<td align='left'>+0.0%</td>
<td align='left'><font color='black'>1.00x</font></td>
</tr>"""
])
def test_sections(self):
"""Report is divided into sections with summaries."""
self.assert_markdown_contains([
"""<details open>
<summary>Regression (1)</summary>""",
"""<details >
<summary>Improvement (2)</summary>""",
"""<details >
<summary>No Changes (2)</summary>""",
"""<details open>
<summary>Added (1)</summary>""",
"""<details open>
<summary>Removed (1)</summary>"""])
self.assert_git_contains([
'Regression (1): \n',
'Improvement (2): \n',
'No Changes (2): \n',
'Added (1): \n',
'Removed (1): \n'])
self.assert_html_contains([
"<th align='left'>Regression (1)</th>",
"<th align='left'>Improvement (2)</th>",
"<th align='left'>No Changes (2)</th>",
"<th align='left'>Added (1)</th>",
"<th align='left'>Removed (1)</th>"])
def test_report_only_changes(self):
"""Leave out tests without significant change."""
rf = ReportFormatter(self.tc, '', '', changes_only=True)
markdown, git, html = rf.markdown(), rf.git(), rf.html()
self.assertNotIn('No Changes', markdown)
self.assertNotIn('AngryPhonebook', markdown)
self.assertNotIn('No Changes', git)
self.assertNotIn('AngryPhonebook', git)
self.assertNotIn('No Changes', html)
self.assertNotIn('AngryPhonebook', html)
class TestReportFormatter(OldAndNewLog):
def setUp(self):
super(TestReportFormatter, self).setUp()
self.tc = TestComparator(self.old_results, self.new_results, 0.05)
self.rf = ReportFormatter(self.tc, '', '', changes_only=False)
self.markdown = self.rf.markdown()
self.git = self.rf.git()
self.html = self.rf.html()
def assert_markdown_contains(self, texts):
self.assert_report_contains(texts, self.markdown)
def assert_git_contains(self, texts):
self.assert_report_contains(texts, self.git)
def assert_html_contains(self, texts):
self.assert_report_contains(texts, self.html)
def test_values(self):
self.assertEquals(
ReportFormatter.values(PerformanceTestResult(
'1,AngryPhonebook,20,10664,12933,11035,576,10884'.split(','))),
('AngryPhonebook', '10664', '12933', '11035', '—')
)
self.assertEquals(
ReportFormatter.values(PerformanceTestResult(
'1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336'
.split(','))),
('AngryPhonebook', '12045', '12045', '12045', '10510336')
)
r1 = PerformanceTestResult(
'1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336'.split(','))
r2 = PerformanceTestResult(
'1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144'.split(','))
self.assertEquals(
ReportFormatter.values(ResultComparison(r1, r2)),
('AngryPhonebook', '12325', '11616', '-5.8%', '1.06x')
)
self.assertEquals(
ReportFormatter.values(ResultComparison(r2, r1)),
('AngryPhonebook', '11616', '12325', '+6.1%', '0.94x')
)
r2.max = r1.min + 1
self.assertEquals(
ReportFormatter.values(ResultComparison(r1, r2))[4],
'1.06x (?)' # is_dubious
)
def test_justified_columns(self):
"""Table columns are all formated with same width, defined by the
longest value.
"""
self.assert_markdown_contains([
'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445',
'Array2D | 335831 | 335831 | +0.0% | 1.00x'])
self.assert_git_contains([
'AnyHashableWithAClass 247027 319065 259056 10250445',
'Array2D 335831 335831 +0.0% 1.00x'])
def test_column_headers(self):
"""Report contains table headers for ResultComparisons and changed
PerformanceTestResults.
"""
performance_test_result = self.tc.added[0]
self.assertEquals(
ReportFormatter.header_for(performance_test_result),
('TEST', 'MIN', 'MAX', 'MEAN', 'MAX_RSS')
)
comparison_result = self.tc.increased[0]
self.assertEquals(
ReportFormatter.header_for(comparison_result),
('TEST', 'OLD', 'NEW', 'DELTA', 'RATIO')
)
self.assert_markdown_contains([
'TEST | OLD | NEW | DELTA | RATIO',
'--- | --- | --- | --- | --- ',
'TEST | MIN | MAX | MEAN | MAX_RSS'])
self.assert_git_contains([
'TEST OLD NEW DELTA RATIO',
'TEST MIN MAX MEAN MAX_RSS'])
self.assert_html_contains([
"""
<th align='left'>OLD</th>
<th align='left'>NEW</th>
<th align='left'>DELTA</th>
<th align='left'>RATIO</th>""",
"""
<th align='left'>MIN</th>
<th align='left'>MAX</th>
<th align='left'>MEAN</th>
<th align='left'>MAX_RSS</th>"""])
def test_emphasize_speedup(self):
"""Emphasize speedup values for regressions and improvements"""
# tests in No Changes don't have emphasized speedup
self.assert_markdown_contains([
'BitCount | 3 | 9 | +199.9% | **0.33x**',
'ByteSwap | 4 | 0 | -100.0% | **4001.00x**',
'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ',
'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**'
])
self.assert_git_contains([
'BitCount 3 9 +199.9% **0.33x**',
'ByteSwap 4 0 -100.0% **4001.00x**',
'AngryPhonebook 10458 10458 +0.0% 1.00x',
'ArrayAppend 23641 20000 -15.4% **1.18x (?)**'
])
self.assert_html_contains([
"""
<tr>
<td align='left'>BitCount</td>
<td align='left'>3</td>
<td align='left'>9</td>
<td align='left'>+199.9%</td>
<td align='left'><font color='red'>0.33x</font></td>
</tr>""",
"""
<tr>
<td align='left'>ByteSwap</td>
<td align='left'>4</td>
<td align='left'>0</td>
<td align='left'>-100.0%</td>
<td align='left'><font color='green'>4001.00x</font></td>
</tr>""",
"""
<tr>
<td align='left'>AngryPhonebook</td>
<td align='left'>10458</td>
<td align='left'>10458</td>
<td align='left'>+0.0%</td>
<td align='left'><font color='black'>1.00x</font></td>
</tr>"""
])
def test_sections(self):
"""Report is divided into sections with summaries."""
self.assert_markdown_contains([
"""<details open>
<summary>Regression (1)</summary>""",
"""<details >
<summary>Improvement (2)</summary>""",
"""<details >
<summary>No Changes (2)</summary>""",
"""<details open>
<summary>Added (1)</summary>""",
"""<details open>
<summary>Removed (1)</summary>"""])
self.assert_git_contains([
'Regression (1): \n',
'Improvement (2): \n',
'No Changes (2): \n',
'Added (1): \n',
'Removed (1): \n'])
self.assert_html_contains([
"<th align='left'>Regression (1)</th>",
"<th align='left'>Improvement (2)</th>",
"<th align='left'>No Changes (2)</th>",
"<th align='left'>Added (1)</th>",
"<th align='left'>Removed (1)</th>"])
def test_report_only_changes(self):
"""Leave out tests without significant change."""
rf = ReportFormatter(self.tc, '', '', changes_only=True)
markdown, git, html = rf.markdown(), rf.git(), rf.html()
self.assertNotIn('No Changes', markdown)
self.assertNotIn('AngryPhonebook', markdown)
self.assertNotIn('No Changes', git)
self.assertNotIn('AngryPhonebook', git)
self.assertNotIn('No Changes', html)
self.assertNotIn('AngryPhonebook', html)
class TestReportFormatter(OldAndNewLog):
def setUp(self):
super(TestReportFormatter, self).setUp()
self.tc = TestComparator(self.old_results, self.new_results, 0.05)
self.rf = ReportFormatter(self.tc, changes_only=False)
self.markdown = self.rf.markdown()
self.git = self.rf.git()
self.html = self.rf.html()
def assert_markdown_contains(self, texts):
self.assert_report_contains(texts, self.markdown)
def assert_git_contains(self, texts):
self.assert_report_contains(texts, self.git)
def assert_html_contains(self, texts):
self.assert_report_contains(texts, self.html)
def test_values(self):
self.assertEquals(
ReportFormatter.values(
PerformanceTestResult(
'1,AngryPhonebook,20,10664,12933,11035,576,10884'.split(
','))),
('AngryPhonebook', '10664', '12933', '11035', '—'))
self.assertEquals(
ReportFormatter.values(
PerformanceTestResult(
'1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336'.
split(','))),
('AngryPhonebook', '12045', '12045', '12045', '10510336'))
r1 = PerformanceTestResult(
'1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336'.split(','))
r2 = PerformanceTestResult(
'1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144'.split(','))
self.assertEquals(ReportFormatter.values(ResultComparison(
r1, r2)), ('AngryPhonebook', '12325', '11616', '-5.8%', '1.06x'))
self.assertEquals(ReportFormatter.values(ResultComparison(
r2, r1)), ('AngryPhonebook', '11616', '12325', '+6.1%', '0.94x'))
r2.max = r1.min + 1
self.assertEquals(
ReportFormatter.values(ResultComparison(r1, r2))[4],
'1.06x (?)' # is_dubious
)
def test_justified_columns(self):
"""Table columns are all formated with same width, defined by the
longest value.
"""
self.assert_markdown_contains([
'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445',
'Array2D | 335831 | 335831 | +0.0% | 1.00x'
])
self.assert_git_contains([
'AnyHashableWithAClass 247027 319065 259056 10250445',
'Array2D 335831 335831 +0.0% 1.00x'
])
def test_column_headers(self):
"""Report contains table headers for ResultComparisons and changed
PerformanceTestResults.
"""
performance_test_result = self.tc.added[0]
self.assertEquals(ReportFormatter.header_for(performance_test_result),
('TEST', 'MIN', 'MAX', 'MEAN', 'MAX_RSS'))
comparison_result = self.tc.increased[0]
self.assertEquals(ReportFormatter.header_for(comparison_result),
('TEST', 'OLD', 'NEW', 'DELTA', 'RATIO'))
self.assert_markdown_contains([
'TEST | OLD | NEW | DELTA | RATIO',
'--- | --- | --- | --- | --- ',
'TEST | MIN | MAX | MEAN | MAX_RSS'
])
self.assert_git_contains([
'TEST OLD NEW DELTA RATIO',
'TEST MIN MAX MEAN MAX_RSS'
])
self.assert_html_contains([
"""
<th align='left'>OLD</th>
<th align='left'>NEW</th>
<th align='left'>DELTA</th>
<th align='left'>RATIO</th>""", """
<th align='left'>MIN</th>
<th align='left'>MAX</th>
<th align='left'>MEAN</th>
<th align='left'>MAX_RSS</th>"""
])
def test_emphasize_speedup(self):
"""Emphasize speedup values for regressions and improvements"""
# tests in No Changes don't have emphasized speedup
self.assert_markdown_contains([
'BitCount | 3 | 9 | +199.9% | **0.33x**',
'ByteSwap | 4 | 0 | -100.0% | **4001.00x**',
'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ',
'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**'
])
self.assert_git_contains([
'BitCount 3 9 +199.9% **0.33x**',
'ByteSwap 4 0 -100.0% **4001.00x**',
'AngryPhonebook 10458 10458 +0.0% 1.00x',
'ArrayAppend 23641 20000 -15.4% **1.18x (?)**'
])
self.assert_html_contains([
"""
<tr>
<td align='left'>BitCount</td>
<td align='left'>3</td>
<td align='left'>9</td>
<td align='left'>+199.9%</td>
<td align='left'><font color='red'>0.33x</font></td>
</tr>""", """
<tr>
<td align='left'>ByteSwap</td>
<td align='left'>4</td>
<td align='left'>0</td>
<td align='left'>-100.0%</td>
<td align='left'><font color='green'>4001.00x</font></td>
</tr>""", """
<tr>
<td align='left'>AngryPhonebook</td>
<td align='left'>10458</td>
<td align='left'>10458</td>
<td align='left'>+0.0%</td>
<td align='left'><font color='black'>1.00x</font></td>
</tr>"""
])
def test_sections(self):
"""Report is divided into sections with summaries."""
self.assert_markdown_contains([
"""<details open>
<summary>Regression (1)</summary>""", """<details >
<summary>Improvement (2)</summary>""", """<details >
<summary>No Changes (2)</summary>""", """<details open>
<summary>Added (1)</summary>""", """<details open>
<summary>Removed (1)</summary>"""
])
self.assert_git_contains([
'Regression (1): \n', 'Improvement (2): \n', 'No Changes (2): \n',
'Added (1): \n', 'Removed (1): \n'
])
self.assert_html_contains([
"<th align='left'>Regression (1)</th>",
"<th align='left'>Improvement (2)</th>",
"<th align='left'>No Changes (2)</th>",
"<th align='left'>Added (1)</th>",
"<th align='left'>Removed (1)</th>"
])
def test_report_only_changes(self):
"""Leave out tests without significant change."""
rf = ReportFormatter(self.tc, changes_only=True)
markdown, git, html = rf.markdown(), rf.git(), rf.html()
self.assertNotIn('No Changes', markdown)
self.assertNotIn('AngryPhonebook', markdown)
self.assertNotIn('No Changes', git)
self.assertNotIn('AngryPhonebook', git)
self.assertNotIn('No Changes', html)
self.assertNotIn('AngryPhonebook', html)
class TestReportFormatter(OldAndNewLog):
def setUp(self):
super(TestReportFormatter, self).setUp()
self.tc = TestComparator(self.old_log, self.new_log, 0.05)
self.rf = ReportFormatter(self.tc, '', '', changes_only=False)
self.markdown = self.rf.markdown()
self.git = self.rf.git()
self.html = self.rf.html()
def assert_markdown_contains(self, texts):
self.assert_report_contains(texts, self.markdown)
def assert_git_contains(self, texts):
self.assert_report_contains(texts, self.git)
def assert_html_contains(self, texts):
self.assert_report_contains(texts, self.html)
def test_justified_columns(self):
"""Table columns are all formated with same width, defined by the
longest value.
"""
self.assert_markdown_contains([
'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445',
'Array2D | 335831 | 335831 | +0.0% | 1.00x'
])
self.assert_git_contains([
'AnyHashableWithAClass 247027 319065 259056 10250445',
'Array2D 335831 335831 +0.0% 1.00x'
])
def test_column_headers(self):
"""Report contains table headers for ResultComparisons and changed
PerformanceTestResults.
"""
self.assert_markdown_contains([
'TEST | OLD | NEW | DELTA | SPEEDUP',
'--- | --- | --- | --- | --- ',
'TEST | MIN | MAX | MEAN | MAX_RSS'
])
self.assert_git_contains([
'TEST OLD NEW DELTA SPEEDUP',
'TEST MIN MAX MEAN MAX_RSS'
])
self.assert_html_contains([
"""
<th align='left'>OLD</th>
<th align='left'>NEW</th>
<th align='left'>DELTA</th>
<th align='left'>SPEEDUP</th>""", """
<th align='left'>MIN</th>
<th align='left'>MAX</th>
<th align='left'>MEAN</th>
<th align='left'>MAX_RSS</th>"""
])
def test_emphasize_speedup(self):
"""Emphasize speedup values for regressions and improvements"""
# tests in No Changes don't have emphasized speedup
self.assert_markdown_contains([
'BitCount | 3 | 9 | +199.9% | **0.33x**',
'ByteSwap | 4 | 0 | -100.0% | **4001.00x**',
'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ',
'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**'
])
self.assert_git_contains([
'BitCount 3 9 +199.9% **0.33x**',
'ByteSwap 4 0 -100.0% **4001.00x**',
'AngryPhonebook 10458 10458 +0.0% 1.00x',
'ArrayAppend 23641 20000 -15.4% **1.18x (?)**'
])
self.assert_html_contains([
"""
<tr>
<td align='left'>BitCount</td>
<td align='left'>3</td>
<td align='left'>9</td>
<td align='left'>+199.9%</td>
<td align='left'><font color='red'>0.33x</font></td>
</tr>""", """
<tr>
<td align='left'>ByteSwap</td>
<td align='left'>4</td>
<td align='left'>0</td>
<td align='left'>-100.0%</td>
<td align='left'><font color='green'>4001.00x</font></td>
</tr>""", """
<tr>
<td align='left'>AngryPhonebook</td>
<td align='left'>10458</td>
<td align='left'>10458</td>
<td align='left'>+0.0%</td>
<td align='left'><font color='black'>1.00x</font></td>
</tr>"""
])
def test_sections(self):
"""Report is divided into sections with summaries."""
self.assert_markdown_contains([
"""<details open>
<summary>Regression (1)</summary>""", """<details >
<summary>Improvement (2)</summary>""", """<details >
<summary>No Changes (2)</summary>""", """<details open>
<summary>Added (1)</summary>""", """<details open>
<summary>Removed (1)</summary>"""
])
self.assert_git_contains([
'Regression (1): \n', 'Improvement (2): \n', 'No Changes (2): \n',
'Added (1): \n', 'Removed (1): \n'
])
self.assert_html_contains([
"<th align='left'>Regression (1)</th>",
"<th align='left'>Improvement (2)</th>",
"<th align='left'>No Changes (2)</th>",
"<th align='left'>Added (1)</th>",
"<th align='left'>Removed (1)</th>"
])
def test_report_only_changes(self):
"""Leave out tests without significant change."""
rf = ReportFormatter(self.tc, '', '', changes_only=True)
markdown, git, html = rf.markdown(), rf.git(), rf.html()
self.assertNotIn('No Changes', markdown)
self.assertNotIn('AngryPhonebook', markdown)
self.assertNotIn('No Changes', git)
self.assertNotIn('AngryPhonebook', git)
self.assertNotIn('No Changes', html)
self.assertNotIn('AngryPhonebook', html)
