def test_parse_delta_quantiles(self):
r = LogParser.results_from_string( # 2-quantile aka. median
"#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,,"
)["B"]
self.assertEqual(
(r.num_samples, r.min, r.median, r.max, r.samples.count),
(1, 101, 101, 101, 1),
)
r = LogParser.results_from_string(
"#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1"
)["B"]
self.assertEqual(
(r.num_samples, r.min, r.median, r.max, r.samples.count),
(2, 101, 101, 102, 2),
)
r = LogParser.results_from_string( # 20-quantiles aka. ventiles
"#,TEST,SAMPLES,MIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8,"
+ "𝚫V9,𝚫VA,𝚫VB,𝚫VC,𝚫VD,𝚫VE,𝚫VF,𝚫VG,𝚫VH,𝚫VI,𝚫VJ,𝚫MAX\n"
+ "202,DropWhileArray,200,214,,,,,,,,,,,,1,,,,,,2,16,464"
)["DropWhileArray"]
self.assertEqual(
(r.num_samples, r.min, r.max, r.samples.count),
# last 3 ventiles were outliers and were excluded from the sample
(200, 214, 215, 18),
)
def test_parse_meta(self):
r = LogParser.results_from_string(
"#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs)," +
"PAGES,ICS,YIELD\n" + "0,B,1,2,2,2,0,2,7,29,15")["B"]
self.assertEqual((r.min, r.mem_pages, r.involuntary_cs, r.yield_count),
(2, 7, 29, 15))
r = LogParser.results_from_string(
"#,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs)," +
"MAX_RSS(B),PAGES,ICS,YIELD\n" +
"0,B,1,3,3,3,0,3,36864,9,50,15")["B"]
self.assertEqual(
(r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss),
(3, 9, 50, 15, 36864),
)
r = LogParser.results_from_string(
"#,TEST,SAMPLES,MIN(μs),MAX(μs),PAGES,ICS,YIELD\n" +
"0,B,1,4,4,8,31,15")["B"]
self.assertEqual((r.min, r.mem_pages, r.involuntary_cs, r.yield_count),
(4, 8, 31, 15))
r = LogParser.results_from_string(
"#,TEST,SAMPLES,MIN(μs),MAX(μs),MAX_RSS(B),PAGES,ICS,YIELD\n" +
"0,B,1,5,5,32768,8,28,15")["B"]
self.assertEqual(
(r.min, r.mem_pages, r.involuntary_cs, r.yield_count, r.max_rss),
(5, 8, 28, 15, 32768),
)
def test_excludes_outliers_from_samples(self):
verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples.
Measuring with scale 2.
Sample 0,455
Measuring with scale 2.
Sample 1,203
Measuring with scale 2.
Sample 2,205
Measuring with scale 2.
Sample 3,207
Measuring with scale 2.
Sample 4,208
Measuring with scale 2.
Sample 5,206
Measuring with scale 2.
Sample 6,205
Measuring with scale 2.
Sample 7,206
Measuring with scale 2.
Sample 8,208
Measuring with scale 2.
Sample 9,184
65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206
"""
parser = LogParser()
result = parser.parse_results(verbose_log.split('\n'))[0]
self.assertEquals(result.num_samples, 10)
self.assertEquals(result.samples.count, 8)
self.assertEquals(len(result.samples.outliers), 2)
def test_excludes_outliers_from_samples(self):
verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples.
Measuring with scale 2.
Sample 0,455
Measuring with scale 2.
Sample 1,203
Measuring with scale 2.
Sample 2,205
Measuring with scale 2.
Sample 3,207
Measuring with scale 2.
Sample 4,208
Measuring with scale 2.
Sample 5,206
Measuring with scale 2.
Sample 6,205
Measuring with scale 2.
Sample 7,206
Measuring with scale 2.
Sample 8,208
Measuring with scale 2.
Sample 9,184
65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206
"""
parser = LogParser()
result = parser.parse_results(verbose_log.split("\n"))[0]
self.assertEqual(result.num_samples, 10)
self.assertEqual(result.samples.count, 8)
self.assertEqual(len(result.samples.outliers), 2)
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self):
"""Ignores header row, empty lines and Totals row."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self):
"""Ignores header row, empty lines and Totals row."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Totals,269
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_formatted_text(self):
"""Parse format that Benchmark_Driver prints to console"""
log = ("""
# TEST SAMPLES MIN(μs) MAX(μs) MEAN(μs) SD(μs) MEDIAN(μs) MAX_RSS(B)
3 Array2D 20 2060 2188 2099 0 2099 20915200
Totals 281""")
parser = LogParser()
results = parser.parse_results(log.splitlines()[1:]) # without 1st \n
self.assertTrue(isinstance(results[0], PerformanceTestResult))
r = results[0]
self.assertEquals(r.name, 'Array2D')
self.assertEquals(r.max_rss, 20915200)
def test_parse_results_verbose(self):
"""Parse multiple performance test results with 2 sample formats:
single line for N = 1; two lines for N > 1.
"""
verbose_log = """--- DATA ---
#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
Running AngryPhonebook for 3 samples.
Measuring with scale 78.
Sample 0,11812
Measuring with scale 90.
Sample 1,13898
Sample 2,11467
1,AngryPhonebook,3,11467,13898,12392,1315,11812
Running Array2D for 3 samples.
SetUp 14444
Sample 0,369900
Yielding after ~369918 μs
Sample 1,381039
Yielding after ~381039 μs
Sample 2,371043
3,Array2D,3,369900,381039,373994,6127,371043
Totals,2"""
parser = LogParser()
results = parser.parse_results(verbose_log.split("\n"))
r = results[0]
self.assertEqual(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
("AngryPhonebook", 11467, 13898, 12392, 1315, 11812),
)
self.assertEqual(r.num_samples, r.samples.num_samples)
self.assertEqual(
results[0].samples.all_samples,
[(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)],
)
self.assertEqual(r.yields, None)
r = results[1]
self.assertEqual(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
("Array2D", 369900, 381039, 373994, 6127, 371043),
)
self.assertEqual(r.setup, 14444)
self.assertEqual(r.num_samples, r.samples.num_samples)
self.assertEqual(
results[1].samples.all_samples,
[(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)],
)
yielded = r.yields[0]
self.assertEqual(yielded.before_sample, 1)
self.assertEqual(yielded.after, 369918)
self.assertEqual(r.yields, [(1, 369918), (2, 381039)])
def test_parse_results_formatted_text(self):
"""Parse format that Benchmark_Driver prints to console"""
log = ("""
# TEST SAMPLES MIN(μs) MAX(μs) MEAN(μs) SD(μs) MEDIAN(μs) MAX_RSS(B)
3 Array2D 20 2060 2188 2099 0 2099 20915200
Totals 281""")
parser = LogParser()
results = parser.parse_results(log.splitlines()[1:]) # without 1st \n
self.assertTrue(isinstance(results[0], PerformanceTestResult))
r = results[0]
self.assertEquals(r.name, 'Array2D')
self.assertEquals(r.max_rss, 20915200)
def test_parse_quantiles(self):
"""Gathers samples from reported quantiles. Handles optional memory."""
r = LogParser.results_from_string(
"""#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs)
1,Ackermann,3,54383,54512,54601""")["Ackermann"]
self.assertEqual([s.runtime for s in r.samples.all_samples],
[54383, 54512, 54601])
r = LogParser.results_from_string(
"""#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B)
1,Ackermann,3,54529,54760,55807,266240""")["Ackermann"]
self.assertEqual([s.runtime for s in r.samples.all_samples],
[54529, 54760, 55807])
self.assertEqual(r.max_rss, 266240)
def test_parse_quantiles(self):
"""Gathers samples from reported quantiles. Handles optional memory."""
r = LogParser.results_from_string(
"""#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs)
1,Ackermann,3,54383,54512,54601""")['Ackermann']
self.assertEquals([s.runtime for s in r.samples.all_samples],
[54383, 54512, 54601])
r = LogParser.results_from_string(
"""#,TEST,SAMPLES,MIN(μs),MEDIAN(μs),MAX(μs),MAX_RSS(B)
1,Ackermann,3,54529,54760,55807,266240""")['Ackermann']
self.assertEquals([s.runtime for s in r.samples.all_samples],
[54529, 54760, 55807])
self.assertEquals(r.max_rss, 266240)
def test_parse_results_verbose(self):
"""Parse multiple performance test results with 2 sample formats:
single line for N = 1; two lines for N > 1.
"""
verbose_log = """--- DATA ---
#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
Running AngryPhonebook for 3 samples.
Measuring with scale 78.
Sample 0,11812
Measuring with scale 90.
Sample 1,13898
Sample 2,11467
1,AngryPhonebook,3,11467,13898,12392,1315,11812
Running Array2D for 3 samples.
SetUp 14444
Sample 0,369900
Yielding after ~369918 μs
Sample 1,381039
Yielding after ~381039 μs
Sample 2,371043
3,Array2D,3,369900,381039,373994,6127,371043
Totals,2"""
parser = LogParser()
results = parser.parse_results(verbose_log.split('\n'))
r = results[0]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('AngryPhonebook', 11467, 13898, 12392, 1315, 11812)
)
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[0].samples.all_samples,
[(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)])
self.assertEquals(r.yields, None)
r = results[1]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('Array2D', 369900, 381039, 373994, 6127, 371043)
)
self.assertEquals(r.setup, 14444)
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[1].samples.all_samples,
[(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)])
yielded = r.yields[0]
self.assertEquals(yielded.before_sample, 1)
self.assertEquals(yielded.after, 369918)
self.assertEquals(r.yields, [(1, 369918), (2, 381039)])
def test_parse_environment_verbose(self):
"""Parse stats about environment in verbose mode."""
verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages)
ICS 1338 - 229 = 1109
VCS 2 - 1 = 1
2,AngryPhonebook,3,11269,11884,11657,338,11820
"""
parser = LogParser()
results = parser.parse_results(verbose_log.split('\n'))
r = results[0]
self.assertEquals(r.max_rss, 32768)
self.assertEquals(r.mem_pages, 8)
self.assertEquals(r.voluntary_cs, 1)
self.assertEquals(r.involuntary_cs, 1109)
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
7,Array.append.Array.Int?,20,10,10,10,0,10
21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11
42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEqual(results[0].name, "Array.append.Array.Int?")
self.assertEqual(results[1].name, "Bridging.NSArray.as!.Array.NSString")
self.assertEqual(results[2].name, "Flatten.Array.Tuple4.lazy.for-in.Reserve")
def test_parse_environment_verbose(self):
"""Parse stats about environment in verbose mode."""
verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages)
ICS 1338 - 229 = 1109
VCS 2 - 1 = 1
2,AngryPhonebook,3,11269,11884,11657,338,11820
"""
parser = LogParser()
results = parser.parse_results(verbose_log.split("\n"))
r = results[0]
self.assertEqual(r.max_rss, 32768)
self.assertEqual(r.mem_pages, 8)
self.assertEqual(r.voluntary_cs, 1)
self.assertEqual(r.involuntary_cs, 1109)
def test_results_from_merge_verbose(self):
"""Parsing verbose log merges all PerformanceTestSamples.
...this should technically be on TestPerformanceTestResult, but it's
easier to write here. ¯\\_(ツ)_/¯"""
concatenated_logs = """
Sample 0,355883
Sample 1,358817
Sample 2,353552
Sample 3,350815
3,Array2D,4,350815,358817,354766,3403,355883
Sample 0,363094
Sample 1,369169
Sample 2,376131
Sample 3,364245
3,Array2D,4,363094,376131,368159,5931,369169"""
results = LogParser.results_from_string(concatenated_logs)
self.assertEqual(list(results.keys()), ["Array2D"])
result = results["Array2D"]
self.assertTrue(isinstance(result, PerformanceTestResult))
self.assertEqual(result.min, 350815)
self.assertEqual(result.max, 376131)
self.assertEqual(result.median, 358817)
self.assertAlmostEqual(result.sd, 8443.37, places=2)
self.assertAlmostEqual(result.mean, 361463.25, places=2)
self.assertEqual(result.num_samples, 8)
samples = result.samples
self.assertTrue(isinstance(samples, PerformanceTestSamples))
self.assertEqual(samples.count, 8)
def test_results_from_merge_verbose(self):
"""Parsing verbose log merges all PerformanceTestSamples.
...this should technically be on TestPerformanceTestResult, but it's
easier to write here. ¯\_(ツ)_/¯"""
concatenated_logs = """
Sample 0,355883
Sample 1,358817
Sample 2,353552
Sample 3,350815
3,Array2D,4,350815,358817,354766,3403,355883
Sample 0,363094
Sample 1,369169
Sample 2,376131
Sample 3,364245
3,Array2D,4,363094,376131,368159,5931,369169"""
results = LogParser.results_from_string(concatenated_logs)
self.assertEquals(results.keys(), ['Array2D'])
result = results['Array2D']
self.assertTrue(isinstance(result, PerformanceTestResult))
self.assertEquals(result.min, 350815)
self.assertEquals(result.max, 376131)
self.assertEquals(result.median, 358817)
self.assertAlmostEquals(result.sd, 8443.37, places=2)
self.assertAlmostEquals(result.mean, 361463.25, places=2)
self.assertEquals(result.num_samples, 8)
samples = result.samples
self.assertTrue(isinstance(samples, PerformanceTestSamples))
self.assertEquals(samples.count, 8)
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
7,Array.append.Array.Int?,20,10,10,10,0,10
21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11
42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'Array.append.Array.Int?')
self.assertEquals(results[1].name,
'Bridging.NSArray.as!.Array.NSString')
self.assertEquals(results[2].name,
'Flatten.Array.Tuple4.lazy.for-in.Reserve')
def test_results_from_merge(self):
"""Parsing concatenated log merges same PerformanceTestResults"""
concatenated_logs = """4,ArrayAppend,20,23641,29000,24990,0,24990
4,ArrayAppend,1,20000,20000,20000,0,20000"""
results = LogParser.results_from_string(concatenated_logs)
self.assertEqual(list(results.keys()), ["ArrayAppend"])
result = results["ArrayAppend"]
self.assertTrue(isinstance(result, PerformanceTestResult))
self.assertEqual(result.min, 20000)
self.assertEqual(result.max, 29000)
def test_results_from_merge(self):
"""Parsing concatenated log merges same PerformanceTestResults"""
concatenated_logs = """4,ArrayAppend,20,23641,29000,24990,0,24990
4,ArrayAppend,1,20000,20000,20000,0,20000"""
results = LogParser.results_from_string(concatenated_logs)
self.assertEquals(results.keys(), ['ArrayAppend'])
result = results['ArrayAppend']
self.assertTrue(isinstance(result, PerformanceTestResult))
self.assertEquals(result.min, 20000)
self.assertEquals(result.max, 29000)
def test_parse_delta_quantiles(self):
r = LogParser.results_from_string( # 2-quantile aka. median
'#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,1,101,,')['B']
self.assertEquals(
(r.num_samples, r.min, r.median, r.max, r.samples.count),
(1, 101, 101, 101, 1))
r = LogParser.results_from_string(
'#,TEST,SAMPLES,MIN(μs),𝚫MEDIAN,𝚫MAX\n0,B,2,101,,1')['B']
self.assertEquals(
(r.num_samples, r.min, r.median, r.max, r.samples.count),
(2, 101, 101, 102, 2))
r = LogParser.results_from_string( # 20-quantiles aka. ventiles
'#,TEST,SAMPLES,MIN(μs),𝚫V1,𝚫V2,𝚫V3,𝚫V4,𝚫V5,𝚫V6,𝚫V7,𝚫V8,' +
'𝚫V9,𝚫VA,𝚫VB,𝚫VC,𝚫VD,𝚫VE,𝚫VF,𝚫VG,𝚫VH,𝚫VI,𝚫VJ,𝚫MAX\n' +
'202,DropWhileArray,200,214,,,,,,,,,,,,1,,,,,,2,16,464'
)['DropWhileArray']
self.assertEquals(
(r.num_samples, r.min, r.max, r.samples.count),
# last 3 ventiles were outliers and were excluded from the sample
(200, 214, 215, 18))
def test_parse_results_verbose(self):
"""Parse multiple performance test results with 2 sample formats:
single line for N = 1; two lines for N > 1.
"""
verbose_log = """--- DATA ---
#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
Running AngryPhonebook for 3 samples.
Measuring with scale 78.
Sample 0,11812
Measuring with scale 90.
Sample 1,13898
Measuring with scale 91.
Sample 2,11467
1,AngryPhonebook,3,11467,13898,12392,1315,11812
Running Array2D for 3 samples.
Sample 0,369900
Sample 1,381039
Sample 2,371043
3,Array2D,3,369900,381039,373994,6127,371043
Totals,2"""
parser = LogParser()
results = parser.parse_results(verbose_log.split('\n'))
r = results[0]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('AngryPhonebook', 11467, 13898, 12392, 1315, 11812))
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[0].samples.all_samples, [(0, 78, 11812),
(1, 90, 13898),
(2, 91, 11467)])
r = results[1]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('Array2D', 369900, 381039, 373994, 6127, 371043))
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[1].samples.all_samples, [(0, 1, 369900),
(1, 1, 381039),
(2, 1, 371043)])
def __init__(self, args, tests=None, _subprocess=None, parser=None):
"""Initialize with command line arguments.
Optional parameters are for injecting dependencies -- used for testing.
"""
self.args = args
self._subprocess = _subprocess or subprocess
self.all_tests = []
self.test_number = {}
self.tests = tests or self._get_tests()
self.parser = parser or LogParser()
self.results = {}
# Set a constant hash seed. Some tests are currently sensitive to
# fluctuations in the number of hash collisions.
os.environ['SWIFT_DETERMINISTIC_HASHING'] = '1'
def test_parse_results_tab_delimited(self):
log = '34\tBitCount\t20\t3\t4\t4\t0\t4'
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_tab_delimited(self):
log = "34\tBitCount\t20\t3\t4\t4\t0\t4"
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEqual(results[0].name, "BitCount")
