def test_parse_delta_quantiles(self): r = LogParser.results_from_string( # 2-quantile aka. median "#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,," )["B"] self.assertEqual( (r.num_samples, r.min, r.median, r.max, r.samples.count), (1, 101, 101, 101, 1), ) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1" )["B"] self.assertEqual( (r.num_samples, r.min, r.median, r.max, r.samples.count), (2, 101, 101, 102, 2), ) r = LogParser.results_from_string( # 20-quantiles aka. ventiles "#,TEST,SAMPLES,MIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8," + "𝚫V9,𝚫VA,𝚫VB,𝚫VC,𝚫VD,𝚫VE,𝚫VF,𝚫VG,𝚫VH,𝚫VI,𝚫VJ,𝚫MAX\n" + "202,DropWhileArray,200,214,,,,,,,,,,,,1,,,,,,2,16,464" )["DropWhileArray"] self.assertEqual( (r.num_samples, r.min, r.max, r.samples.count), # last 3 ventiles were outliers and were excluded from the sample (200, 214, 215, 18), )
def test_parse_meta(self): r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs)," + "PAGES,ICS,YIELD\n" + "0,B,1,2,2,2,0,2,7,29,15")["B"] self.assertEqual((r.min, r.mem_pages, r.involuntary_cs, r.yield_count), (2, 7, 29, 15)) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs)," + "MAX_RSS(B),PAGES,ICS,YIELD\n" + "0,B,1,3,3,3,0,3,36864,9,50,15")["B"] self.assertEqual( (r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss), (3, 9, 50, 15, 36864), ) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),PAGES,ICS,YIELD\n" + "0,B,1,4,4,8,31,15")["B"] self.assertEqual((r.min, r.mem_pages, r.involuntary_cs, r.yield_count), (4, 8, 31, 15)) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),MAX_RSS(B),PAGES,ICS,YIELD\n" + "0,B,1,5,5,32768,8,28,15")["B"] self.assertEqual( (r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss), (5, 8, 28, 15, 32768), )
def test_excludes_outliers_from_samples(self): verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples. Measuring with scale 2. Sample 0,455 Measuring with scale 2. Sample 1,203 Measuring with scale 2. Sample 2,205 Measuring with scale 2. Sample 3,207 Measuring with scale 2. Sample 4,208 Measuring with scale 2. Sample 5,206 Measuring with scale 2. Sample 6,205 Measuring with scale 2. Sample 7,206 Measuring with scale 2. Sample 8,208 Measuring with scale 2. Sample 9,184 65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206 """ parser = LogParser() result = parser.parse_results(verbose_log.split('\n'))[0] self.assertEquals(result.num_samples, 10) self.assertEquals(result.samples.count, 8) self.assertEquals(len(result.samples.outliers), 2)
def test_excludes_outliers_from_samples(self): verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples. Measuring with scale 2. Sample 0,455 Measuring with scale 2. Sample 1,203 Measuring with scale 2. Sample 2,205 Measuring with scale 2. Sample 3,207 Measuring with scale 2. Sample 4,208 Measuring with scale 2. Sample 5,206 Measuring with scale 2. Sample 6,205 Measuring with scale 2. Sample 7,206 Measuring with scale 2. Sample 8,208 Measuring with scale 2. Sample 9,184 65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206 """ parser = LogParser() result = parser.parse_results(verbose_log.split("\n"))[0] self.assertEqual(result.num_samples, 10) self.assertEqual(result.samples.count, 8) self.assertEqual(len(result.samples.outliers), 2)
def test_parse_results_csv(self): """Ignores uknown lines, extracts data from supported formats.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 34,BitCount,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self): """Ignores header row, empty lines and Totals row.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 34,BitCount,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self): """Ignores header row, empty lines and Totals row.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 34,BitCount,20,3,4,4,0,4 Totals,269 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_formatted_text(self): """Parse format that Benchmark_Driver prints to console""" log = (""" # TEST SAMPLES MIN(μs) MAX(μs) MEAN(μs) SD(μs) MEDIAN(μs) MAX_RSS(B) 3 Array2D 20 2060 2188 2099 0 2099 20915200 Totals 281""") parser = LogParser() results = parser.parse_results(log.splitlines()[1:]) # without 1st \n self.assertTrue(isinstance(results[0], PerformanceTestResult)) r = results[0] self.assertEquals(r.name, 'Array2D') self.assertEquals(r.max_rss, 20915200)
def test_parse_results_verbose(self): """Parse multiple performance test results with 2 sample formats: single line for N = 1; two lines for N > 1. """ verbose_log = """--- DATA --- #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) Running AngryPhonebook for 3 samples. Measuring with scale 78. Sample 0,11812 Measuring with scale 90. Sample 1,13898 Sample 2,11467 1,AngryPhonebook,3,11467,13898,12392,1315,11812 Running Array2D for 3 samples. SetUp 14444 Sample 0,369900 Yielding after ~369918 μs Sample 1,381039 Yielding after ~381039 μs Sample 2,371043 3,Array2D,3,369900,381039,373994,6127,371043 Totals,2""" parser = LogParser() results = parser.parse_results(verbose_log.split("\n")) r = results[0] self.assertEqual( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ("AngryPhonebook", 11467, 13898, 12392, 1315, 11812), ) self.assertEqual(r.num_samples, r.samples.num_samples) self.assertEqual( results[0].samples.all_samples, [(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)], ) self.assertEqual(r.yields, None) r = results[1] self.assertEqual( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ("Array2D", 369900, 381039, 373994, 6127, 371043), ) self.assertEqual(r.setup, 14444) self.assertEqual(r.num_samples, r.samples.num_samples) self.assertEqual( results[1].samples.all_samples, [(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)], ) yielded = r.yields[0] self.assertEqual(yielded.before_sample, 1) self.assertEqual(yielded.after, 369918) self.assertEqual(r.yields, [(1, 369918), (2, 381039)])
def test_parse_quantiles(self): """Gathers samples from reported quantiles. Handles optional memory.""" r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs) 1,Ackermann,3,54383,54512,54601""")["Ackermann"] self.assertEqual([s.runtime for s in r.samples.all_samples], [54383, 54512, 54601]) r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B) 1,Ackermann,3,54529,54760,55807,266240""")["Ackermann"] self.assertEqual([s.runtime for s in r.samples.all_samples], [54529, 54760, 55807]) self.assertEqual(r.max_rss, 266240)
def test_parse_quantiles(self): """Gathers samples from reported quantiles. Handles optional memory.""" r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs) 1,Ackermann,3,54383,54512,54601""")['Ackermann'] self.assertEquals([s.runtime for s in r.samples.all_samples], [54383, 54512, 54601]) r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B) 1,Ackermann,3,54529,54760,55807,266240""")['Ackermann'] self.assertEquals([s.runtime for s in r.samples.all_samples], [54529, 54760, 55807]) self.assertEquals(r.max_rss, 266240)
def test_parse_results_verbose(self): """Parse multiple performance test results with 2 sample formats: single line for N = 1; two lines for N > 1. """ verbose_log = """--- DATA --- #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) Running AngryPhonebook for 3 samples. Measuring with scale 78. Sample 0,11812 Measuring with scale 90. Sample 1,13898 Sample 2,11467 1,AngryPhonebook,3,11467,13898,12392,1315,11812 Running Array2D for 3 samples. SetUp 14444 Sample 0,369900 Yielding after ~369918 μs Sample 1,381039 Yielding after ~381039 μs Sample 2,371043 3,Array2D,3,369900,381039,373994,6127,371043 Totals,2""" parser = LogParser() results = parser.parse_results(verbose_log.split('\n')) r = results[0] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('AngryPhonebook', 11467, 13898, 12392, 1315, 11812) ) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[0].samples.all_samples, [(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)]) self.assertEquals(r.yields, None) r = results[1] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('Array2D', 369900, 381039, 373994, 6127, 371043) ) self.assertEquals(r.setup, 14444) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[1].samples.all_samples, [(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)]) yielded = r.yields[0] self.assertEquals(yielded.before_sample, 1) self.assertEquals(yielded.after, 369918) self.assertEquals(r.yields, [(1, 369918), (2, 381039)])
def test_parse_environment_verbose(self): """Parse stats about environment in verbose mode.""" verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages) ICS 1338 - 229 = 1109 VCS 2 - 1 = 1 2,AngryPhonebook,3,11269,11884,11657,338,11820 """ parser = LogParser() results = parser.parse_results(verbose_log.split('\n')) r = results[0] self.assertEquals(r.max_rss, 32768) self.assertEquals(r.mem_pages, 8) self.assertEquals(r.voluntary_cs, 1) self.assertEquals(r.involuntary_cs, 1109)
def test_parse_results_csv(self): """Ignores uknown lines, extracts data from supported formats.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 7,Array.append.Array.Int?,20,10,10,10,0,10 21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11 42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEqual(results[0].name, "Array.append.Array.Int?") self.assertEqual(results[1].name, "Bridging.NSArray.as!.Array.NSString") self.assertEqual(results[2].name, "Flatten.Array.Tuple4.lazy.for-in.Reserve")
def test_parse_environment_verbose(self): """Parse stats about environment in verbose mode.""" verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages) ICS 1338 - 229 = 1109 VCS 2 - 1 = 1 2,AngryPhonebook,3,11269,11884,11657,338,11820 """ parser = LogParser() results = parser.parse_results(verbose_log.split("\n")) r = results[0] self.assertEqual(r.max_rss, 32768) self.assertEqual(r.mem_pages, 8) self.assertEqual(r.voluntary_cs, 1) self.assertEqual(r.involuntary_cs, 1109)
def test_results_from_merge_verbose(self): """Parsing verbose log merges all PerformanceTestSamples. ...this should technically be on TestPerformanceTestResult, but it's easier to write here. ¯\\_(ツ)_/¯""" concatenated_logs = """ Sample 0,355883 Sample 1,358817 Sample 2,353552 Sample 3,350815 3,Array2D,4,350815,358817,354766,3403,355883 Sample 0,363094 Sample 1,369169 Sample 2,376131 Sample 3,364245 3,Array2D,4,363094,376131,368159,5931,369169""" results = LogParser.results_from_string(concatenated_logs) self.assertEqual(list(results.keys()), ["Array2D"]) result = results["Array2D"] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEqual(result.min, 350815) self.assertEqual(result.max, 376131) self.assertEqual(result.median, 358817) self.assertAlmostEqual(result.sd, 8443.37, places=2) self.assertAlmostEqual(result.mean, 361463.25, places=2) self.assertEqual(result.num_samples, 8) samples = result.samples self.assertTrue(isinstance(samples, PerformanceTestSamples)) self.assertEqual(samples.count, 8)
def test_results_from_merge_verbose(self): """Parsing verbose log merges all PerformanceTestSamples. ...this should technically be on TestPerformanceTestResult, but it's easier to write here. ¯\_(ツ)_/¯""" concatenated_logs = """ Sample 0,355883 Sample 1,358817 Sample 2,353552 Sample 3,350815 3,Array2D,4,350815,358817,354766,3403,355883 Sample 0,363094 Sample 1,369169 Sample 2,376131 Sample 3,364245 3,Array2D,4,363094,376131,368159,5931,369169""" results = LogParser.results_from_string(concatenated_logs) self.assertEquals(results.keys(), ['Array2D']) result = results['Array2D'] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEquals(result.min, 350815) self.assertEquals(result.max, 376131) self.assertEquals(result.median, 358817) self.assertAlmostEquals(result.sd, 8443.37, places=2) self.assertAlmostEquals(result.mean, 361463.25, places=2) self.assertEquals(result.num_samples, 8) samples = result.samples self.assertTrue(isinstance(samples, PerformanceTestSamples)) self.assertEquals(samples.count, 8)
def test_parse_results_csv(self): """Ignores uknown lines, extracts data from supported formats.""" log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) 7,Array.append.Array.Int?,20,10,10,10,0,10 21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11 42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4 Total performance tests executed: 1 """ parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'Array.append.Array.Int?') self.assertEquals(results[1].name, 'Bridging.NSArray.as!.Array.NSString') self.assertEquals(results[2].name, 'Flatten.Array.Tuple4.lazy.for-in.Reserve')
def test_results_from_merge(self): """Parsing concatenated log merges same PerformanceTestResults""" concatenated_logs = """4,ArrayAppend,20,23641,29000,24990,0,24990 4,ArrayAppend,1,20000,20000,20000,0,20000""" results = LogParser.results_from_string(concatenated_logs) self.assertEqual(list(results.keys()), ["ArrayAppend"]) result = results["ArrayAppend"] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEqual(result.min, 20000) self.assertEqual(result.max, 29000)
def test_results_from_merge(self): """Parsing concatenated log merges same PerformanceTestResults""" concatenated_logs = """4,ArrayAppend,20,23641,29000,24990,0,24990 4,ArrayAppend,1,20000,20000,20000,0,20000""" results = LogParser.results_from_string(concatenated_logs) self.assertEquals(results.keys(), ['ArrayAppend']) result = results['ArrayAppend'] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEquals(result.min, 20000) self.assertEquals(result.max, 29000)
def test_parse_delta_quantiles(self): r = LogParser.results_from_string( # 2-quantile aka. median '#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,,')['B'] self.assertEquals( (r.num_samples, r.min, r.median, r.max, r.samples.count), (1, 101, 101, 101, 1)) r = LogParser.results_from_string( '#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1')['B'] self.assertEquals( (r.num_samples, r.min, r.median, r.max, r.samples.count), (2, 101, 101, 102, 2)) r = LogParser.results_from_string( # 20-quantiles aka. ventiles '#,TEST,SAMPLES,MIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8,' + '𝚫V9,𝚫VA,𝚫VB,𝚫VC,𝚫VD,𝚫VE,𝚫VF,𝚫VG,𝚫VH,𝚫VI,𝚫VJ,𝚫MAX\n' + '202,DropWhileArray,200,214,,,,,,,,,,,,1,,,,,,2,16,464' )['DropWhileArray'] self.assertEquals( (r.num_samples, r.min, r.max, r.samples.count), # last 3 ventiles were outliers and were excluded from the sample (200, 214, 215, 18))
def test_parse_results_verbose(self): """Parse multiple performance test results with 2 sample formats: single line for N = 1; two lines for N > 1. """ verbose_log = """--- DATA --- #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) Running AngryPhonebook for 3 samples. Measuring with scale 78. Sample 0,11812 Measuring with scale 90. Sample 1,13898 Measuring with scale 91. Sample 2,11467 1,AngryPhonebook,3,11467,13898,12392,1315,11812 Running Array2D for 3 samples. Sample 0,369900 Sample 1,381039 Sample 2,371043 3,Array2D,3,369900,381039,373994,6127,371043 Totals,2""" parser = LogParser() results = parser.parse_results(verbose_log.split('\n')) r = results[0] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('AngryPhonebook', 11467, 13898, 12392, 1315, 11812)) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[0].samples.all_samples, [(0, 78, 11812), (1, 90, 13898), (2, 91, 11467)]) r = results[1] self.assertEquals( (r.name, r.min, r.max, int(r.mean), int(r.sd), r.median), ('Array2D', 369900, 381039, 373994, 6127, 371043)) self.assertEquals(r.num_samples, r.samples.num_samples) self.assertEquals(results[1].samples.all_samples, [(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)])
def __init__(self, args, tests=None, _subprocess=None, parser=None): """Initialize with command line arguments. Optional parameters are for injecting dependencies -- used for testing. """ self.args = args self._subprocess = _subprocess or subprocess self.all_tests = [] self.test_number = {} self.tests = tests or self._get_tests() self.parser = parser or LogParser() self.results = {} # Set a constant hash seed. Some tests are currently sensitive to # fluctuations in the number of hash collisions. os.environ['SWIFT_DETERMINISTIC_HASHING'] = '1'
def test_parse_results_tab_delimited(self): log = '34\tBitCount\t20\t3\t4\t4\t0\t4' parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_tab_delimited(self): log = "34\tBitCount\t20\t3\t4\t4\t0\t4" parser = LogParser() results = parser.parse_results(log.splitlines()) self.assertTrue(isinstance(results[0], PerformanceTestResult)) self.assertEqual(results[0].name, "BitCount")