def test_values(self): self.assertEqual( ReportFormatter.values( PerformanceTestResult( "1,AngryPhonebook,20,10664,12933,11035,576,10884".split( ","))), ("AngryPhonebook", "10664", "12933", "11035", "—"), ) self.assertEqual( ReportFormatter.values( PerformanceTestResult( "1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336". split(","))), ("AngryPhonebook", "12045", "12045", "12045", "10510336"), ) r1 = PerformanceTestResult( "1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336".split(",")) r2 = PerformanceTestResult( "1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144".split(",")) self.assertEqual( ReportFormatter.values(ResultComparison(r1, r2)), ("AngryPhonebook", "12325", "11616", "-5.8%", "1.06x"), ) self.assertEqual( ReportFormatter.values(ResultComparison(r2, r1)), ("AngryPhonebook", "11616", "12325", "+6.1%", "0.94x"), ) r2.max = r1.min + 1 self.assertEqual( ReportFormatter.values(ResultComparison(r1, r2))[4], "1.06x (?)", # is_dubious )
def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ performance_test_result = self.tc.added[0] self.assertEqual( ReportFormatter.header_for(performance_test_result), ("TEST", "MIN", "MAX", "MEAN", "MAX_RSS"), ) comparison_result = self.tc.increased[0] self.assertEqual( ReportFormatter.header_for(comparison_result), ("TEST", "OLD", "NEW", "DELTA", "RATIO"), ) self.assert_markdown_contains([ "TEST | OLD | NEW | DELTA | RATIO", ":--- | ---: | ---: | ---: | ---: ", "TEST | MIN | MAX | MEAN | MAX_RSS", ]) self.assert_git_contains([ "TEST OLD NEW DELTA RATIO", "TEST MIN MAX MEAN MAX_RSS", ]) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>RATIO</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>""", ])
def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ performance_test_result = self.tc.added[0] self.assertEquals( ReportFormatter.header_for(performance_test_result), ('TEST', 'MIN', 'MAX', 'MEAN', 'MAX_RSS') ) comparison_result = self.tc.increased[0] self.assertEquals( ReportFormatter.header_for(comparison_result), ('TEST', 'OLD', 'NEW', 'DELTA', 'RATIO') ) self.assert_markdown_contains([ 'TEST | OLD | NEW | DELTA | RATIO', '--- | --- | --- | --- | --- ', 'TEST | MIN | MAX | MEAN | MAX_RSS']) self.assert_git_contains([ 'TEST OLD NEW DELTA RATIO', 'TEST MIN MAX MEAN MAX_RSS']) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>RATIO</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>"""])
def setUp(self): super(TestReportFormatter, self).setUp() self.tc = TestComparator(self.old_results, self.new_results, 0.05) self.rf = ReportFormatter(self.tc, changes_only=False) self.markdown = self.rf.markdown() self.git = self.rf.git() self.html = self.rf.html()
def test_single_table_report(self): """Single table report has inline headers and no elaborate sections.""" self.tc.removed = [] # test handling empty section rf = ReportFormatter(self.tc, changes_only=True, single_table=True) markdown = rf.markdown() self.assertNotIn('<details', markdown) # no sections self.assertNotIn('\n\n', markdown) # table must not be broken self.assertNotIn('Removed', markdown) self.assert_report_contains([ '\n**Regression** ', '| **OLD**', '| **NEW**', '| **DELTA**', '| **RATIO**', '\n**Added** ', '| **MIN**', '| **MAX**', '| **MEAN**', '| **MAX_RSS**' ], markdown) # Single delimiter row: self.assertIn('\n:---', markdown) # first column is left aligned self.assertEqual(markdown.count('| ---:'), 4) # other, right aligned # Separator before every inline header (new section): self.assertEqual(markdown.count(' | | | | '), 2) git = rf.git() self.assertNotIn('): \n', git) # no sections self.assertNotIn('REMOVED', git) self.assert_report_contains([ '\nREGRESSION ', ' OLD ', ' NEW ', ' DELTA ', ' RATIO ', '\n\nADDED ', ' MIN ', ' MAX ', ' MEAN ', ' MAX_RSS ' ], git) # Separator before every inline header (new section): self.assertEqual(git.count('\n\n'), 2)
def test_values(self): self.assertEquals( ReportFormatter.values(PerformanceTestResult( '1,AngryPhonebook,20,10664,12933,11035,576,10884'.split(','))), ('AngryPhonebook', '10664', '12933', '11035', '—') ) self.assertEquals( ReportFormatter.values(PerformanceTestResult( '1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336' .split(','))), ('AngryPhonebook', '12045', '12045', '12045', '10510336') ) r1 = PerformanceTestResult( '1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336'.split(',')) r2 = PerformanceTestResult( '1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144'.split(',')) self.assertEquals( ReportFormatter.values(ResultComparison(r1, r2)), ('AngryPhonebook', '12325', '11616', '-5.8%', '1.06x') ) self.assertEquals( ReportFormatter.values(ResultComparison(r2, r1)), ('AngryPhonebook', '11616', '12325', '+6.1%', '0.94x') ) r2.max = r1.min + 1 self.assertEquals( ReportFormatter.values(ResultComparison(r1, r2))[4], '1.06x (?)' # is_dubious )
def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ performance_test_result = self.tc.added[0] self.assertEqual(ReportFormatter.header_for(performance_test_result), ('TEST', 'MIN', 'MAX', 'MEAN', 'MAX_RSS')) comparison_result = self.tc.increased[0] self.assertEqual(ReportFormatter.header_for(comparison_result), ('TEST', 'OLD', 'NEW', 'DELTA', 'RATIO')) self.assert_markdown_contains([ 'TEST | OLD | NEW | DELTA | RATIO', ':--- | ---: | ---: | ---: | ---: ', 'TEST | MIN | MAX | MEAN | MAX_RSS' ]) self.assert_git_contains([ 'TEST OLD NEW DELTA RATIO', 'TEST MIN MAX MEAN MAX_RSS' ]) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>RATIO</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>""" ])
def test_values(self): self.assertEqual( ReportFormatter.values( PerformanceTestResult( '1,AngryPhonebook,20,10664,12933,11035,576,10884'.split( ','))), ('AngryPhonebook', '10664', '12933', '11035', '—')) self.assertEqual( ReportFormatter.values( PerformanceTestResult( '1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336'. split(','))), ('AngryPhonebook', '12045', '12045', '12045', '10510336')) r1 = PerformanceTestResult( '1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336'.split(',')) r2 = PerformanceTestResult( '1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144'.split(',')) self.assertEqual(ReportFormatter.values(ResultComparison( r1, r2)), ('AngryPhonebook', '12325', '11616', '-5.8%', '1.06x')) self.assertEqual(ReportFormatter.values(ResultComparison( r2, r1)), ('AngryPhonebook', '11616', '12325', '+6.1%', '0.94x')) r2.max = r1.min + 1 self.assertEqual( ReportFormatter.values(ResultComparison(r1, r2))[4], '1.06x (?)' # is_dubious )
def test_report_only_changes(self): """Leave out tests without significant change.""" rf = ReportFormatter(self.tc, '', '', changes_only=True) markdown, git, html = rf.markdown(), rf.git(), rf.html() self.assertNotIn('No Changes', markdown) self.assertNotIn('AngryPhonebook', markdown) self.assertNotIn('No Changes', git) self.assertNotIn('AngryPhonebook', git) self.assertNotIn('No Changes', html) self.assertNotIn('AngryPhonebook', html)
def test_report_only_changes(self): """Leave out tests without significant change.""" rf = ReportFormatter(self.tc, changes_only=True) markdown, git, html = rf.markdown(), rf.git(), rf.html() self.assertNotIn("No Changes", markdown) self.assertNotIn("AngryPhonebook", markdown) self.assertNotIn("No Changes", git) self.assertNotIn("AngryPhonebook", git) self.assertNotIn("No Changes", html) self.assertNotIn("AngryPhonebook", html)
def test_single_table_report(self): """Single table report has inline headers and no elaborate sections.""" self.tc.removed = [] # test handling empty section rf = ReportFormatter(self.tc, changes_only=True, single_table=True) markdown = rf.markdown() self.assertNotIn("<details", markdown) # no sections self.assertNotIn("\n\n", markdown) # table must not be broken self.assertNotIn("Removed", markdown) self.assert_report_contains( [ "\n**Regression** ", "| **OLD**", "| **NEW**", "| **DELTA**", "| **RATIO**", "\n**Added** ", "| **MIN**", "| **MAX**", "| **MEAN**", "| **MAX_RSS**", ], markdown, ) # Single delimiter row: self.assertIn("\n:---", markdown) # first column is left aligned self.assertEqual(markdown.count("| ---:"), 4) # other, right aligned # Separator before every inline header (new section): self.assertEqual(markdown.count(" | | | | "), 2) git = rf.git() self.assertNotIn("): \n", git) # no sections self.assertNotIn("REMOVED", git) self.assert_report_contains( [ "\nREGRESSION ", " OLD ", " NEW ", " DELTA ", " RATIO ", "\n\nADDED ", " MIN ", " MAX ", " MEAN ", " MAX_RSS ", ], git, ) # Separator before every inline header (new section): self.assertEqual(git.count("\n\n"), 2)
def setUp(self): super(TestReportFormatter, self).setUp() self.tc = TestComparator(self.old_log, self.new_log, 0.05) self.rf = ReportFormatter(self.tc, '', '', changes_only=False) self.markdown = self.rf.markdown() self.git = self.rf.git() self.html = self.rf.html()
class TestReportFormatter(OldAndNewLog): def setUp(self): super(TestReportFormatter, self).setUp() self.tc = TestComparator(self.old_results, self.new_results, 0.05) self.rf = ReportFormatter(self.tc, changes_only=False) self.markdown = self.rf.markdown() self.git = self.rf.git() self.html = self.rf.html() def assert_markdown_contains(self, texts): self.assert_report_contains(texts, self.markdown) def assert_git_contains(self, texts): self.assert_report_contains(texts, self.git) def assert_html_contains(self, texts): self.assert_report_contains(texts, self.html) def test_values(self): self.assertEqual( ReportFormatter.values( PerformanceTestResult( "1,AngryPhonebook,20,10664,12933,11035,576,10884".split( ","))), ("AngryPhonebook", "10664", "12933", "11035", "—"), ) self.assertEqual( ReportFormatter.values( PerformanceTestResult( "1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336". split(","))), ("AngryPhonebook", "12045", "12045", "12045", "10510336"), ) r1 = PerformanceTestResult( "1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336".split(",")) r2 = PerformanceTestResult( "1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144".split(",")) self.assertEqual( ReportFormatter.values(ResultComparison(r1, r2)), ("AngryPhonebook", "12325", "11616", "-5.8%", "1.06x"), ) self.assertEqual( ReportFormatter.values(ResultComparison(r2, r1)), ("AngryPhonebook", "11616", "12325", "+6.1%", "0.94x"), ) r2.max = r1.min + 1 self.assertEqual( ReportFormatter.values(ResultComparison(r1, r2))[4], "1.06x (?)", # is_dubious ) def test_justified_columns(self): """Table columns are all formated with same width, defined by the longest value. """ self.assert_markdown_contains([ "AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445", "Array2D | 335831 | 335831 | +0.0% | 1.00x", ]) self.assert_git_contains([ "AnyHashableWithAClass 247027 319065 259056 10250445", "Array2D 335831 335831 +0.0% 1.00x", ]) def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ performance_test_result = self.tc.added[0] self.assertEqual( ReportFormatter.header_for(performance_test_result), ("TEST", "MIN", "MAX", "MEAN", "MAX_RSS"), ) comparison_result = self.tc.increased[0] self.assertEqual( ReportFormatter.header_for(comparison_result), ("TEST", "OLD", "NEW", "DELTA", "RATIO"), ) self.assert_markdown_contains([ "TEST | OLD | NEW | DELTA | RATIO", ":--- | ---: | ---: | ---: | ---: ", "TEST | MIN | MAX | MEAN | MAX_RSS", ]) self.assert_git_contains([ "TEST OLD NEW DELTA RATIO", "TEST MIN MAX MEAN MAX_RSS", ]) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>RATIO</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>""", ]) def test_emphasize_speedup(self): """Emphasize speedup values for regressions and improvements""" # tests in No Changes don't have emphasized speedup self.assert_markdown_contains([ "BitCount | 3 | 9 | +199.9% | **0.33x**", "ByteSwap | 4 | 0 | -100.0% | **4001.00x**", "AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ", "ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**", ]) self.assert_git_contains([ "BitCount 3 9 +199.9% **0.33x**", "ByteSwap 4 0 -100.0% **4001.00x**", "AngryPhonebook 10458 10458 +0.0% 1.00x", "ArrayAppend 23641 20000 -15.4% **1.18x (?)**", ]) self.assert_html_contains([ """ <tr> <td align='left'>BitCount</td> <td align='left'>3</td> <td align='left'>9</td> <td align='left'>+199.9%</td> <td align='left'><font color='red'>0.33x</font></td> </tr>""", """ <tr> <td align='left'>ByteSwap</td> <td align='left'>4</td> <td align='left'>0</td> <td align='left'>-100.0%</td> <td align='left'><font color='green'>4001.00x</font></td> </tr>""", """ <tr> <td align='left'>AngryPhonebook</td> <td align='left'>10458</td> <td align='left'>10458</td> <td align='left'>+0.0%</td> <td align='left'><font color='black'>1.00x</font></td> </tr>""", ]) def test_sections(self): """Report is divided into sections with summaries.""" self.assert_markdown_contains([ """<details open> <summary>Regression (1)</summary>""", """<details > <summary>Improvement (2)</summary>""", """<details > <summary>No Changes (2)</summary>""", """<details open> <summary>Added (1)</summary>""", """<details open> <summary>Removed (1)</summary>""", ]) self.assert_git_contains([ "Regression (1): \n", "Improvement (2): \n", "No Changes (2): \n", "Added (1): \n", "Removed (1): \n", ]) self.assert_html_contains([ "<th align='left'>Regression (1)</th>", "<th align='left'>Improvement (2)</th>", "<th align='left'>No Changes (2)</th>", "<th align='left'>Added (1)</th>", "<th align='left'>Removed (1)</th>", ]) def test_report_only_changes(self): """Leave out tests without significant change.""" rf = ReportFormatter(self.tc, changes_only=True) markdown, git, html = rf.markdown(), rf.git(), rf.html() self.assertNotIn("No Changes", markdown) self.assertNotIn("AngryPhonebook", markdown) self.assertNotIn("No Changes", git) self.assertNotIn("AngryPhonebook", git) self.assertNotIn("No Changes", html) self.assertNotIn("AngryPhonebook", html) def test_single_table_report(self): """Single table report has inline headers and no elaborate sections.""" self.tc.removed = [] # test handling empty section rf = ReportFormatter(self.tc, changes_only=True, single_table=True) markdown = rf.markdown() self.assertNotIn("<details", markdown) # no sections self.assertNotIn("\n\n", markdown) # table must not be broken self.assertNotIn("Removed", markdown) self.assert_report_contains( [ "\n**Regression** ", "| **OLD**", "| **NEW**", "| **DELTA**", "| **RATIO**", "\n**Added** ", "| **MIN**", "| **MAX**", "| **MEAN**", "| **MAX_RSS**", ], markdown, ) # Single delimiter row: self.assertIn("\n:---", markdown) # first column is left aligned self.assertEqual(markdown.count("| ---:"), 4) # other, right aligned # Separator before every inline header (new section): self.assertEqual(markdown.count(" | | | | "), 2) git = rf.git() self.assertNotIn("): \n", git) # no sections self.assertNotIn("REMOVED", git) self.assert_report_contains( [ "\nREGRESSION ", " OLD ", " NEW ", " DELTA ", " RATIO ", "\n\nADDED ", " MIN ", " MAX ", " MEAN ", " MAX_RSS ", ], git, ) # Separator before every inline header (new section): self.assertEqual(git.count("\n\n"), 2)
class TestReportFormatter(OldAndNewLog): def setUp(self): super(TestReportFormatter, self).setUp() self.tc = TestComparator(self.old_results, self.new_results, 0.05) self.rf = ReportFormatter(self.tc, changes_only=False) self.markdown = self.rf.markdown() self.git = self.rf.git() self.html = self.rf.html() def assert_markdown_contains(self, texts): self.assert_report_contains(texts, self.markdown) def assert_git_contains(self, texts): self.assert_report_contains(texts, self.git) def assert_html_contains(self, texts): self.assert_report_contains(texts, self.html) def test_values(self): self.assertEquals( ReportFormatter.values( PerformanceTestResult( '1,AngryPhonebook,20,10664,12933,11035,576,10884'.split( ','))), ('AngryPhonebook', '10664', '12933', '11035', '—')) self.assertEquals( ReportFormatter.values( PerformanceTestResult( '1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336'. split(','))), ('AngryPhonebook', '12045', '12045', '12045', '10510336')) r1 = PerformanceTestResult( '1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336'.split(',')) r2 = PerformanceTestResult( '1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144'.split(',')) self.assertEquals(ReportFormatter.values(ResultComparison( r1, r2)), ('AngryPhonebook', '12325', '11616', '-5.8%', '1.06x')) self.assertEquals(ReportFormatter.values(ResultComparison( r2, r1)), ('AngryPhonebook', '11616', '12325', '+6.1%', '0.94x')) r2.max = r1.min + 1 self.assertEquals( ReportFormatter.values(ResultComparison(r1, r2))[4], '1.06x (?)' # is_dubious ) def test_justified_columns(self): """Table columns are all formated with same width, defined by the longest value. """ self.assert_markdown_contains([ 'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445', 'Array2D | 335831 | 335831 | +0.0% | 1.00x' ]) self.assert_git_contains([ 'AnyHashableWithAClass 247027 319065 259056 10250445', 'Array2D 335831 335831 +0.0% 1.00x' ]) def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ performance_test_result = self.tc.added[0] self.assertEquals(ReportFormatter.header_for(performance_test_result), ('TEST', 'MIN', 'MAX', 'MEAN', 'MAX_RSS')) comparison_result = self.tc.increased[0] self.assertEquals(ReportFormatter.header_for(comparison_result), ('TEST', 'OLD', 'NEW', 'DELTA', 'RATIO')) self.assert_markdown_contains([ 'TEST | OLD | NEW | DELTA | RATIO', '--- | --- | --- | --- | --- ', 'TEST | MIN | MAX | MEAN | MAX_RSS' ]) self.assert_git_contains([ 'TEST OLD NEW DELTA RATIO', 'TEST MIN MAX MEAN MAX_RSS' ]) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>RATIO</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>""" ]) def test_emphasize_speedup(self): """Emphasize speedup values for regressions and improvements""" # tests in No Changes don't have emphasized speedup self.assert_markdown_contains([ 'BitCount | 3 | 9 | +199.9% | **0.33x**', 'ByteSwap | 4 | 0 | -100.0% | **4001.00x**', 'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ', 'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**' ]) self.assert_git_contains([ 'BitCount 3 9 +199.9% **0.33x**', 'ByteSwap 4 0 -100.0% **4001.00x**', 'AngryPhonebook 10458 10458 +0.0% 1.00x', 'ArrayAppend 23641 20000 -15.4% **1.18x (?)**' ]) self.assert_html_contains([ """ <tr> <td align='left'>BitCount</td> <td align='left'>3</td> <td align='left'>9</td> <td align='left'>+199.9%</td> <td align='left'><font color='red'>0.33x</font></td> </tr>""", """ <tr> <td align='left'>ByteSwap</td> <td align='left'>4</td> <td align='left'>0</td> <td align='left'>-100.0%</td> <td align='left'><font color='green'>4001.00x</font></td> </tr>""", """ <tr> <td align='left'>AngryPhonebook</td> <td align='left'>10458</td> <td align='left'>10458</td> <td align='left'>+0.0%</td> <td align='left'><font color='black'>1.00x</font></td> </tr>""" ]) def test_sections(self): """Report is divided into sections with summaries.""" self.assert_markdown_contains([ """<details open> <summary>Regression (1)</summary>""", """<details > <summary>Improvement (2)</summary>""", """<details > <summary>No Changes (2)</summary>""", """<details open> <summary>Added (1)</summary>""", """<details open> <summary>Removed (1)</summary>""" ]) self.assert_git_contains([ 'Regression (1): \n', 'Improvement (2): \n', 'No Changes (2): \n', 'Added (1): \n', 'Removed (1): \n' ]) self.assert_html_contains([ "<th align='left'>Regression (1)</th>", "<th align='left'>Improvement (2)</th>", "<th align='left'>No Changes (2)</th>", "<th align='left'>Added (1)</th>", "<th align='left'>Removed (1)</th>" ]) def test_report_only_changes(self): """Leave out tests without significant change.""" rf = ReportFormatter(self.tc, changes_only=True) markdown, git, html = rf.markdown(), rf.git(), rf.html() self.assertNotIn('No Changes', markdown) self.assertNotIn('AngryPhonebook', markdown) self.assertNotIn('No Changes', git) self.assertNotIn('AngryPhonebook', git) self.assertNotIn('No Changes', html) self.assertNotIn('AngryPhonebook', html)
class TestReportFormatter(OldAndNewLog): def setUp(self): super(TestReportFormatter, self).setUp() self.tc = TestComparator(self.old_results, self.new_results, 0.05) self.rf = ReportFormatter(self.tc, '', '', changes_only=False) self.markdown = self.rf.markdown() self.git = self.rf.git() self.html = self.rf.html() def assert_markdown_contains(self, texts): self.assert_report_contains(texts, self.markdown) def assert_git_contains(self, texts): self.assert_report_contains(texts, self.git) def assert_html_contains(self, texts): self.assert_report_contains(texts, self.html) def test_values(self): self.assertEquals( ReportFormatter.values(PerformanceTestResult( '1,AngryPhonebook,20,10664,12933,11035,576,10884'.split(','))), ('AngryPhonebook', '10664', '12933', '11035', '—') ) self.assertEquals( ReportFormatter.values(PerformanceTestResult( '1,AngryPhonebook,1,12045,12045,12045,0,12045,10510336' .split(','))), ('AngryPhonebook', '12045', '12045', '12045', '10510336') ) r1 = PerformanceTestResult( '1,AngryPhonebook,1,12325,12325,12325,0,12325,10510336'.split(',')) r2 = PerformanceTestResult( '1,AngryPhonebook,1,11616,11616,11616,0,11616,10502144'.split(',')) self.assertEquals( ReportFormatter.values(ResultComparison(r1, r2)), ('AngryPhonebook', '12325', '11616', '-5.8%', '1.06x') ) self.assertEquals( ReportFormatter.values(ResultComparison(r2, r1)), ('AngryPhonebook', '11616', '12325', '+6.1%', '0.94x') ) r2.max = r1.min + 1 self.assertEquals( ReportFormatter.values(ResultComparison(r1, r2))[4], '1.06x (?)' # is_dubious ) def test_justified_columns(self): """Table columns are all formated with same width, defined by the longest value. """ self.assert_markdown_contains([ 'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445', 'Array2D | 335831 | 335831 | +0.0% | 1.00x']) self.assert_git_contains([ 'AnyHashableWithAClass 247027 319065 259056 10250445', 'Array2D 335831 335831 +0.0% 1.00x']) def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ performance_test_result = self.tc.added[0] self.assertEquals( ReportFormatter.header_for(performance_test_result), ('TEST', 'MIN', 'MAX', 'MEAN', 'MAX_RSS') ) comparison_result = self.tc.increased[0] self.assertEquals( ReportFormatter.header_for(comparison_result), ('TEST', 'OLD', 'NEW', 'DELTA', 'RATIO') ) self.assert_markdown_contains([ 'TEST | OLD | NEW | DELTA | RATIO', '--- | --- | --- | --- | --- ', 'TEST | MIN | MAX | MEAN | MAX_RSS']) self.assert_git_contains([ 'TEST OLD NEW DELTA RATIO', 'TEST MIN MAX MEAN MAX_RSS']) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>RATIO</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>"""]) def test_emphasize_speedup(self): """Emphasize speedup values for regressions and improvements""" # tests in No Changes don't have emphasized speedup self.assert_markdown_contains([ 'BitCount | 3 | 9 | +199.9% | **0.33x**', 'ByteSwap | 4 | 0 | -100.0% | **4001.00x**', 'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ', 'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**' ]) self.assert_git_contains([ 'BitCount 3 9 +199.9% **0.33x**', 'ByteSwap 4 0 -100.0% **4001.00x**', 'AngryPhonebook 10458 10458 +0.0% 1.00x', 'ArrayAppend 23641 20000 -15.4% **1.18x (?)**' ]) self.assert_html_contains([ """ <tr> <td align='left'>BitCount</td> <td align='left'>3</td> <td align='left'>9</td> <td align='left'>+199.9%</td> <td align='left'><font color='red'>0.33x</font></td> </tr>""", """ <tr> <td align='left'>ByteSwap</td> <td align='left'>4</td> <td align='left'>0</td> <td align='left'>-100.0%</td> <td align='left'><font color='green'>4001.00x</font></td> </tr>""", """ <tr> <td align='left'>AngryPhonebook</td> <td align='left'>10458</td> <td align='left'>10458</td> <td align='left'>+0.0%</td> <td align='left'><font color='black'>1.00x</font></td> </tr>""" ]) def test_sections(self): """Report is divided into sections with summaries.""" self.assert_markdown_contains([ """<details open> <summary>Regression (1)</summary>""", """<details > <summary>Improvement (2)</summary>""", """<details > <summary>No Changes (2)</summary>""", """<details open> <summary>Added (1)</summary>""", """<details open> <summary>Removed (1)</summary>"""]) self.assert_git_contains([ 'Regression (1): \n', 'Improvement (2): \n', 'No Changes (2): \n', 'Added (1): \n', 'Removed (1): \n']) self.assert_html_contains([ "<th align='left'>Regression (1)</th>", "<th align='left'>Improvement (2)</th>", "<th align='left'>No Changes (2)</th>", "<th align='left'>Added (1)</th>", "<th align='left'>Removed (1)</th>"]) def test_report_only_changes(self): """Leave out tests without significant change.""" rf = ReportFormatter(self.tc, '', '', changes_only=True) markdown, git, html = rf.markdown(), rf.git(), rf.html() self.assertNotIn('No Changes', markdown) self.assertNotIn('AngryPhonebook', markdown) self.assertNotIn('No Changes', git) self.assertNotIn('AngryPhonebook', git) self.assertNotIn('No Changes', html) self.assertNotIn('AngryPhonebook', html)
class TestReportFormatter(OldAndNewLog): def setUp(self): super(TestReportFormatter, self).setUp() self.tc = TestComparator(self.old_log, self.new_log, 0.05) self.rf = ReportFormatter(self.tc, '', '', changes_only=False) self.markdown = self.rf.markdown() self.git = self.rf.git() self.html = self.rf.html() def assert_markdown_contains(self, texts): self.assert_report_contains(texts, self.markdown) def assert_git_contains(self, texts): self.assert_report_contains(texts, self.git) def assert_html_contains(self, texts): self.assert_report_contains(texts, self.html) def test_justified_columns(self): """Table columns are all formated with same width, defined by the longest value. """ self.assert_markdown_contains([ 'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445', 'Array2D | 335831 | 335831 | +0.0% | 1.00x']) self.assert_git_contains([ 'AnyHashableWithAClass 247027 319065 259056 10250445', 'Array2D 335831 335831 +0.0% 1.00x']) def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ self.assert_markdown_contains([ 'TEST | OLD | NEW | DELTA | SPEEDUP', '--- | --- | --- | --- | --- ', 'TEST | MIN | MAX | MEAN | MAX_RSS']) self.assert_git_contains([ 'TEST OLD NEW DELTA SPEEDUP', 'TEST MIN MAX MEAN MAX_RSS']) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>SPEEDUP</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>"""]) def test_emphasize_speedup(self): """Emphasize speedup values for regressions and improvements""" # tests in No Changes don't have emphasized speedup self.assert_markdown_contains([ 'BitCount | 3 | 9 | +199.9% | **0.33x**', 'ByteSwap | 4 | 0 | -100.0% | **4001.00x**', 'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ', 'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**' ]) self.assert_git_contains([ 'BitCount 3 9 +199.9% **0.33x**', 'ByteSwap 4 0 -100.0% **4001.00x**', 'AngryPhonebook 10458 10458 +0.0% 1.00x', 'ArrayAppend 23641 20000 -15.4% **1.18x (?)**' ]) self.assert_html_contains([ """ <tr> <td align='left'>BitCount</td> <td align='left'>3</td> <td align='left'>9</td> <td align='left'>+199.9%</td> <td align='left'><font color='red'>0.33x</font></td> </tr>""", """ <tr> <td align='left'>ByteSwap</td> <td align='left'>4</td> <td align='left'>0</td> <td align='left'>-100.0%</td> <td align='left'><font color='green'>4001.00x</font></td> </tr>""", """ <tr> <td align='left'>AngryPhonebook</td> <td align='left'>10458</td> <td align='left'>10458</td> <td align='left'>+0.0%</td> <td align='left'><font color='black'>1.00x</font></td> </tr>""" ]) def test_sections(self): """Report is divided into sections with summaries.""" self.assert_markdown_contains([ """<details open> <summary>Regression (1)</summary>""", """<details > <summary>Improvement (2)</summary>""", """<details > <summary>No Changes (2)</summary>""", """<details open> <summary>Added (1)</summary>""", """<details open> <summary>Removed (1)</summary>"""]) self.assert_git_contains([ 'Regression (1): \n', 'Improvement (2): \n', 'No Changes (2): \n', 'Added (1): \n', 'Removed (1): \n']) self.assert_html_contains([ "<th align='left'>Regression (1)</th>", "<th align='left'>Improvement (2)</th>", "<th align='left'>No Changes (2)</th>", "<th align='left'>Added (1)</th>", "<th align='left'>Removed (1)</th>"]) def test_report_only_changes(self): """Leave out tests without significant change.""" rf = ReportFormatter(self.tc, '', '', changes_only=True) markdown, git, html = rf.markdown(), rf.git(), rf.html() self.assertNotIn('No Changes', markdown) self.assertNotIn('AngryPhonebook', markdown) self.assertNotIn('No Changes', git) self.assertNotIn('AngryPhonebook', git) self.assertNotIn('No Changes', html) self.assertNotIn('AngryPhonebook', html)
class TestReportFormatter(OldAndNewLog): def setUp(self): super(TestReportFormatter, self).setUp() self.tc = TestComparator(self.old_log, self.new_log, 0.05) self.rf = ReportFormatter(self.tc, '', '', changes_only=False) self.markdown = self.rf.markdown() self.git = self.rf.git() self.html = self.rf.html() def assert_markdown_contains(self, texts): self.assert_report_contains(texts, self.markdown) def assert_git_contains(self, texts): self.assert_report_contains(texts, self.git) def assert_html_contains(self, texts): self.assert_report_contains(texts, self.html) def test_justified_columns(self): """Table columns are all formated with same width, defined by the longest value. """ self.assert_markdown_contains([ 'AnyHashableWithAClass | 247027 | 319065 | 259056 | 10250445', 'Array2D | 335831 | 335831 | +0.0% | 1.00x' ]) self.assert_git_contains([ 'AnyHashableWithAClass 247027 319065 259056 10250445', 'Array2D 335831 335831 +0.0% 1.00x' ]) def test_column_headers(self): """Report contains table headers for ResultComparisons and changed PerformanceTestResults. """ self.assert_markdown_contains([ 'TEST | OLD | NEW | DELTA | SPEEDUP', '--- | --- | --- | --- | --- ', 'TEST | MIN | MAX | MEAN | MAX_RSS' ]) self.assert_git_contains([ 'TEST OLD NEW DELTA SPEEDUP', 'TEST MIN MAX MEAN MAX_RSS' ]) self.assert_html_contains([ """ <th align='left'>OLD</th> <th align='left'>NEW</th> <th align='left'>DELTA</th> <th align='left'>SPEEDUP</th>""", """ <th align='left'>MIN</th> <th align='left'>MAX</th> <th align='left'>MEAN</th> <th align='left'>MAX_RSS</th>""" ]) def test_emphasize_speedup(self): """Emphasize speedup values for regressions and improvements""" # tests in No Changes don't have emphasized speedup self.assert_markdown_contains([ 'BitCount | 3 | 9 | +199.9% | **0.33x**', 'ByteSwap | 4 | 0 | -100.0% | **4001.00x**', 'AngryPhonebook | 10458 | 10458 | +0.0% | 1.00x ', 'ArrayAppend | 23641 | 20000 | -15.4% | **1.18x (?)**' ]) self.assert_git_contains([ 'BitCount 3 9 +199.9% **0.33x**', 'ByteSwap 4 0 -100.0% **4001.00x**', 'AngryPhonebook 10458 10458 +0.0% 1.00x', 'ArrayAppend 23641 20000 -15.4% **1.18x (?)**' ]) self.assert_html_contains([ """ <tr> <td align='left'>BitCount</td> <td align='left'>3</td> <td align='left'>9</td> <td align='left'>+199.9%</td> <td align='left'><font color='red'>0.33x</font></td> </tr>""", """ <tr> <td align='left'>ByteSwap</td> <td align='left'>4</td> <td align='left'>0</td> <td align='left'>-100.0%</td> <td align='left'><font color='green'>4001.00x</font></td> </tr>""", """ <tr> <td align='left'>AngryPhonebook</td> <td align='left'>10458</td> <td align='left'>10458</td> <td align='left'>+0.0%</td> <td align='left'><font color='black'>1.00x</font></td> </tr>""" ]) def test_sections(self): """Report is divided into sections with summaries.""" self.assert_markdown_contains([ """<details open> <summary>Regression (1)</summary>""", """<details > <summary>Improvement (2)</summary>""", """<details > <summary>No Changes (2)</summary>""", """<details open> <summary>Added (1)</summary>""", """<details open> <summary>Removed (1)</summary>""" ]) self.assert_git_contains([ 'Regression (1): \n', 'Improvement (2): \n', 'No Changes (2): \n', 'Added (1): \n', 'Removed (1): \n' ]) self.assert_html_contains([ "<th align='left'>Regression (1)</th>", "<th align='left'>Improvement (2)</th>", "<th align='left'>No Changes (2)</th>", "<th align='left'>Added (1)</th>", "<th align='left'>Removed (1)</th>" ]) def test_report_only_changes(self): """Leave out tests without significant change.""" rf = ReportFormatter(self.tc, '', '', changes_only=True) markdown, git, html = rf.markdown(), rf.git(), rf.html() self.assertNotIn('No Changes', markdown) self.assertNotIn('AngryPhonebook', markdown) self.assertNotIn('No Changes', git) self.assertNotIn('AngryPhonebook', git) self.assertNotIn('No Changes', html) self.assertNotIn('AngryPhonebook', html)