def test_parse_delta_quantiles(self): r = LogParser.results_from_string( # 2-quantile aka. median "#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,," )["B"] self.assertEqual( (r.num_samples, r.min, r.median, r.max, r.samples.count), (1, 101, 101, 101, 1), ) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1" )["B"] self.assertEqual( (r.num_samples, r.min, r.median, r.max, r.samples.count), (2, 101, 101, 102, 2), ) r = LogParser.results_from_string( # 20-quantiles aka. ventiles "#,TEST,SAMPLES,MIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8," + "𝚫V9,𝚫VA,𝚫VB,𝚫VC,𝚫VD,𝚫VE,𝚫VF,𝚫VG,𝚫VH,𝚫VI,𝚫VJ,𝚫MAX\n" + "202,DropWhileArray,200,214,,,,,,,,,,,,1,,,,,,2,16,464" )["DropWhileArray"] self.assertEqual( (r.num_samples, r.min, r.max, r.samples.count), # last 3 ventiles were outliers and were excluded from the sample (200, 214, 215, 18), )
def test_parse_meta(self): r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs)," + "PAGES,ICS,YIELD\n" + "0,B,1,2,2,2,0,2,7,29,15")["B"] self.assertEqual((r.min, r.mem_pages, r.involuntary_cs, r.yield_count), (2, 7, 29, 15)) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs)," + "MAX_RSS(B),PAGES,ICS,YIELD\n" + "0,B,1,3,3,3,0,3,36864,9,50,15")["B"] self.assertEqual( (r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss), (3, 9, 50, 15, 36864), ) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),PAGES,ICS,YIELD\n" + "0,B,1,4,4,8,31,15")["B"] self.assertEqual((r.min, r.mem_pages, r.involuntary_cs, r.yield_count), (4, 8, 31, 15)) r = LogParser.results_from_string( "#,TEST,SAMPLES,MIN(μs),MAX(μs),MAX_RSS(B),PAGES,ICS,YIELD\n" + "0,B,1,5,5,32768,8,28,15")["B"] self.assertEqual( (r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss), (5, 8, 28, 15, 32768), )
def test_parse_quantiles(self): """Gathers samples from reported quantiles. Handles optional memory.""" r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs) 1,Ackermann,3,54383,54512,54601""")["Ackermann"] self.assertEqual([s.runtime for s in r.samples.all_samples], [54383, 54512, 54601]) r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B) 1,Ackermann,3,54529,54760,55807,266240""")["Ackermann"] self.assertEqual([s.runtime for s in r.samples.all_samples], [54529, 54760, 55807]) self.assertEqual(r.max_rss, 266240)
def test_parse_quantiles(self): """Gathers samples from reported quantiles. Handles optional memory.""" r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs) 1,Ackermann,3,54383,54512,54601""")['Ackermann'] self.assertEquals([s.runtime for s in r.samples.all_samples], [54383, 54512, 54601]) r = LogParser.results_from_string( """#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B) 1,Ackermann,3,54529,54760,55807,266240""")['Ackermann'] self.assertEquals([s.runtime for s in r.samples.all_samples], [54529, 54760, 55807]) self.assertEquals(r.max_rss, 266240)
def test_results_from_merge_verbose(self): """Parsing verbose log merges all PerformanceTestSamples. ...this should technically be on TestPerformanceTestResult, but it's easier to write here. ¯\\_(ツ)_/¯""" concatenated_logs = """ Sample 0,355883 Sample 1,358817 Sample 2,353552 Sample 3,350815 3,Array2D,4,350815,358817,354766,3403,355883 Sample 0,363094 Sample 1,369169 Sample 2,376131 Sample 3,364245 3,Array2D,4,363094,376131,368159,5931,369169""" results = LogParser.results_from_string(concatenated_logs) self.assertEqual(list(results.keys()), ["Array2D"]) result = results["Array2D"] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEqual(result.min, 350815) self.assertEqual(result.max, 376131) self.assertEqual(result.median, 358817) self.assertAlmostEqual(result.sd, 8443.37, places=2) self.assertAlmostEqual(result.mean, 361463.25, places=2) self.assertEqual(result.num_samples, 8) samples = result.samples self.assertTrue(isinstance(samples, PerformanceTestSamples)) self.assertEqual(samples.count, 8)
def test_results_from_merge_verbose(self): """Parsing verbose log merges all PerformanceTestSamples. ...this should technically be on TestPerformanceTestResult, but it's easier to write here. ¯\_(ツ)_/¯""" concatenated_logs = """ Sample 0,355883 Sample 1,358817 Sample 2,353552 Sample 3,350815 3,Array2D,4,350815,358817,354766,3403,355883 Sample 0,363094 Sample 1,369169 Sample 2,376131 Sample 3,364245 3,Array2D,4,363094,376131,368159,5931,369169""" results = LogParser.results_from_string(concatenated_logs) self.assertEquals(results.keys(), ['Array2D']) result = results['Array2D'] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEquals(result.min, 350815) self.assertEquals(result.max, 376131) self.assertEquals(result.median, 358817) self.assertAlmostEquals(result.sd, 8443.37, places=2) self.assertAlmostEquals(result.mean, 361463.25, places=2) self.assertEquals(result.num_samples, 8) samples = result.samples self.assertTrue(isinstance(samples, PerformanceTestSamples)) self.assertEquals(samples.count, 8)
def test_results_from_merge(self): """Parsing concatenated log merges same PerformanceTestResults""" concatenated_logs = """4,ArrayAppend,20,23641,29000,24990,0,24990 4,ArrayAppend,1,20000,20000,20000,0,20000""" results = LogParser.results_from_string(concatenated_logs) self.assertEqual(list(results.keys()), ["ArrayAppend"]) result = results["ArrayAppend"] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEqual(result.min, 20000) self.assertEqual(result.max, 29000)
def test_results_from_merge(self): """Parsing concatenated log merges same PerformanceTestResults""" concatenated_logs = """4,ArrayAppend,20,23641,29000,24990,0,24990 4,ArrayAppend,1,20000,20000,20000,0,20000""" results = LogParser.results_from_string(concatenated_logs) self.assertEquals(results.keys(), ['ArrayAppend']) result = results['ArrayAppend'] self.assertTrue(isinstance(result, PerformanceTestResult)) self.assertEquals(result.min, 20000) self.assertEquals(result.max, 29000)
def test_parse_delta_quantiles(self): r = LogParser.results_from_string( # 2-quantile aka. median '#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,,')['B'] self.assertEquals( (r.num_samples, r.min, r.median, r.max, r.samples.count), (1, 101, 101, 101, 1)) r = LogParser.results_from_string( '#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1')['B'] self.assertEquals( (r.num_samples, r.min, r.median, r.max, r.samples.count), (2, 101, 101, 102, 2)) r = LogParser.results_from_string( # 20-quantiles aka. ventiles '#,TEST,SAMPLES,MIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8,' + '𝚫V9,𝚫VA,𝚫VB,𝚫VC,𝚫VD,𝚫VE,𝚫VF,𝚫VG,𝚫VH,𝚫VI,𝚫VJ,𝚫MAX\n' + '202,DropWhileArray,200,214,,,,,,,,,,,,1,,,,,,2,16,464' )['DropWhileArray'] self.assertEquals( (r.num_samples, r.min, r.max, r.samples.count), # last 3 ventiles were outliers and were excluded from the sample (200, 214, 215, 18))