def test_excludes_outliers_from_samples(self): verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples. Measuring with scale 2. Sample 0,455 Measuring with scale 2. Sample 1,203 Measuring with scale 2. Sample 2,205 Measuring with scale 2. Sample 3,207 Measuring with scale 2. Sample 4,208 Measuring with scale 2. Sample 5,206 Measuring with scale 2. Sample 6,205 Measuring with scale 2. Sample 7,206 Measuring with scale 2. Sample 8,208 Measuring with scale 2. Sample 9,184 65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206 """ parser = LogParser() result = parser.parse_results(verbose_log.split('\n'))[0] self.assertEquals(result.num_samples, 10) self.assertEquals(result.samples.count, 8) self.assertEquals(len(result.samples.outliers), 2)
def test_excludes_outliers_from_samples(self): verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples. Measuring with scale 2. Sample 0,455 Measuring with scale 2. Sample 1,203 Measuring with scale 2. Sample 2,205 Measuring with scale 2. Sample 3,207 Measuring with scale 2. Sample 4,208 Measuring with scale 2. Sample 5,206 Measuring with scale 2. Sample 6,205 Measuring with scale 2. Sample 7,206 Measuring with scale 2. Sample 8,208 Measuring with scale 2. Sample 9,184 65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206 """ parser = LogParser() result = parser.parse_results(verbose_log.split("\n"))[0] self.assertEqual(result.num_samples, 10) self.assertEqual(result.samples.count, 8) self.assertEqual(len(result.samples.outliers), 2)
def test_parse_results_csv(self): """Ignores header row, empty lines and Totals row.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 34,BitCount,20,3,4,4,0,4 Totals,269 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self): """Ignores uknown lines, extracts data from supported formats.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 34,BitCount,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self): """Ignores header row, empty lines and Totals row.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 34,BitCount,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_formatted_text(self): """Parse format that Benchmark_Driver prints to console""" log = (""" # TEST SAMPLES MIN(μs) MAX(μs) MEAN(μs) SD(μs) MEDIAN(μs) MAX_RSS(B) 3 Array2D 20 2060 2188 2099 0 2099 20915200 Totals 281""") parser = LogParser() results = parser.parse_results(log.splitlines()[1:]) # without 1st \n self.assertTrue(isinstance(results[0], PerformanceTestResult)) r = results[0] self.assertEquals(r.name, 'Array2D') self.assertEquals(r.max_rss, 20915200)
def test_parse_results_verbose(self): """Parse multiple performance test results with 2 sample formats: single line for N = 1; two lines for N > 1. """ verbose_log = """--- DATA --- #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) Running AngryPhonebook for 3 samples. Measuring with scale 78. Sample 0,11812 Measuring with scale 90. Sample 1,13898 Sample 2,11467 1,AngryPhonebook,3,11467,13898,12392,1315,11812 Running Array2D for 3 samples. SetUp 14444 Sample 0,369900 Yielding after ~369918 μs Sample 1,381039 Yielding after ~381039 μs Sample 2,371043 3,Array2D,3,369900,381039,373994,6127,371043 Totals,2""" parser = LogParser() results = parser.parse_results(verbose_log.split("\n")) r = results[0] self.assertEqual( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ("AngryPhonebook", 11467, 13898, 12392, 1315, 11812), ) self.assertEqual(r.num_samples, r.samples.num_samples) self.assertEqual( results[0].samples.all_samples, [(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)], ) self.assertEqual(r.yields, None) r = results[1] self.assertEqual( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ("Array2D", 369900, 381039, 373994, 6127, 371043), ) self.assertEqual(r.setup, 14444) self.assertEqual(r.num_samples, r.samples.num_samples) self.assertEqual( results[1].samples.all_samples, [(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)], ) yielded = r.yields[0] self.assertEqual(yielded.before_sample, 1) self.assertEqual(yielded.after, 369918) self.assertEqual(r.yields, [(1, 369918), (2, 381039)])
def test_parse_results_verbose(self): """Parse multiple performance test results with 2 sample formats: single line for N = 1; two lines for N > 1. """ verbose_log = """--- DATA --- #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) Running AngryPhonebook for 3 samples. Measuring with scale 78. Sample 0,11812 Measuring with scale 90. Sample 1,13898 Sample 2,11467 1,AngryPhonebook,3,11467,13898,12392,1315,11812 Running Array2D for 3 samples. SetUp 14444 Sample 0,369900 Yielding after ~369918 μs Sample 1,381039 Yielding after ~381039 μs Sample 2,371043 3,Array2D,3,369900,381039,373994,6127,371043 Totals,2""" parser = LogParser() results = parser.parse_results(verbose_log.split('\n')) r = results[0] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('AngryPhonebook', 11467, 13898, 12392, 1315, 11812) ) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[0].samples.all_samples, [(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)]) self.assertEquals(r.yields, None) r = results[1] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('Array2D', 369900, 381039, 373994, 6127, 371043) ) self.assertEquals(r.setup, 14444) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[1].samples.all_samples, [(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)]) yielded = r.yields[0] self.assertEquals(yielded.before_sample, 1) self.assertEquals(yielded.after, 369918) self.assertEquals(r.yields, [(1, 369918), (2, 381039)])
def test_parse_results_csv(self): """Ignores uknown lines, extracts data from supported formats.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 7,Array.append.Array.Int?,20,10,10,10,0,10 21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11 42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEqual(results[0].name, "Array.append.Array.Int?") self.assertEqual(results[1].name, "Bridging.NSArray.as!.Array.NSString") self.assertEqual(results[2].name, "Flatten.Array.Tuple4.lazy.for-in.Reserve")
def test_parse_environment_verbose(self): """Parse stats about environment in verbose mode.""" verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages) ICS 1338 - 229 = 1109 VCS 2 - 1 = 1 2,AngryPhonebook,3,11269,11884,11657,338,11820 """ parser = LogParser() results = parser.parse_results(verbose_log.split('\n')) r = results[0] self.assertEquals(r.max_rss, 32768) self.assertEquals(r.mem_pages, 8) self.assertEquals(r.voluntary_cs, 1) self.assertEquals(r.involuntary_cs, 1109)
def test_parse_environment_verbose(self): """Parse stats about environment in verbose mode.""" verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages) ICS 1338 - 229 = 1109 VCS 2 - 1 = 1 2,AngryPhonebook,3,11269,11884,11657,338,11820 """ parser = LogParser() results = parser.parse_results(verbose_log.split("\n")) r = results[0] self.assertEqual(r.max_rss, 32768) self.assertEqual(r.mem_pages, 8) self.assertEqual(r.voluntary_cs, 1) self.assertEqual(r.involuntary_cs, 1109)
def test_parse_results_csv(self): """Ignores uknown lines, extracts data from supported formats.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 7,Array.append.Array.Int?,20,10,10,10,0,10 21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11 42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'Array.append.Array.Int?') self.assertEquals(results[1].name, 'Bridging.NSArray.as!.Array.NSString') self.assertEquals(results[2].name, 'Flatten.Array.Tuple4.lazy.for-in.Reserve')
def test_parse_results_verbose(self): """Parse multiple performance test results with 2 sample formats: single line for N = 1; two lines for N > 1. """ verbose_log = """--- DATA --- #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) Running AngryPhonebook for 3 samples. Measuring with scale 78. Sample 0,11812 Measuring with scale 90. Sample 1,13898 Measuring with scale 91. Sample 2,11467 1,AngryPhonebook,3,11467,13898,12392,1315,11812 Running Array2D for 3 samples. Sample 0,369900 Sample 1,381039 Sample 2,371043 3,Array2D,3,369900,381039,373994,6127,371043 Totals,2""" parser = LogParser() results = parser.parse_results(verbose_log.split('\n')) r = results[0] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('AngryPhonebook', 11467, 13898, 12392, 1315, 11812)) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[0].samples.all_samples, [(0, 78, 11812), (1, 90, 13898), (2, 91, 11467)]) r = results[1] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('Array2D', 369900, 381039, 373994, 6127, 371043)) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[1].samples.all_samples, [(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)])
def test_parse_results_tab_delimited(self): log = '34\tBitCount\t20\t3\t4\t4\t0\t4' parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_tab_delimited(self): log = "34\tBitCount\t20\t3\t4\t4\t0\t4" parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEqual(results[0].name, "BitCount")