def test_excludes_outliers_from_samples(self):
verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples.
Measuring with scale 2.
Sample 0,455
Measuring with scale 2.
Sample 1,203
Measuring with scale 2.
Sample 2,205
Measuring with scale 2.
Sample 3,207
Measuring with scale 2.
Sample 4,208
Measuring with scale 2.
Sample 5,206
Measuring with scale 2.
Sample 6,205
Measuring with scale 2.
Sample 7,206
Measuring with scale 2.
Sample 8,208
Measuring with scale 2.
Sample 9,184
65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206
"""
parser = LogParser()
result = parser.parse_results(verbose_log.split('\n'))[0]
self.assertEquals(result.num_samples, 10)
self.assertEquals(result.samples.count, 8)
self.assertEquals(len(result.samples.outliers), 2)
def test_excludes_outliers_from_samples(self):
verbose_log = """Running DropFirstAnySeqCntRangeLazy for 10 samples.
Measuring with scale 2.
Sample 0,455
Measuring with scale 2.
Sample 1,203
Measuring with scale 2.
Sample 2,205
Measuring with scale 2.
Sample 3,207
Measuring with scale 2.
Sample 4,208
Measuring with scale 2.
Sample 5,206
Measuring with scale 2.
Sample 6,205
Measuring with scale 2.
Sample 7,206
Measuring with scale 2.
Sample 8,208
Measuring with scale 2.
Sample 9,184
65,DropFirstAnySeqCntRangeLazy,10,184,455,228,79,206
"""
parser = LogParser()
result = parser.parse_results(verbose_log.split("\n"))[0]
self.assertEqual(result.num_samples, 10)
self.assertEqual(result.samples.count, 8)
self.assertEqual(len(result.samples.outliers), 2)
def test_parse_results_csv(self):
"""Ignores header row, empty lines and Totals row."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Totals,269
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self):
"""Ignores header row, empty lines and Totals row."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
34,BitCount,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_formatted_text(self):
"""Parse format that Benchmark_Driver prints to console"""
log = ("""
# TEST SAMPLES MIN(μs) MAX(μs) MEAN(μs) SD(μs) MEDIAN(μs) MAX_RSS(B)
3 Array2D 20 2060 2188 2099 0 2099 20915200
Totals 281""")
parser = LogParser()
results = parser.parse_results(log.splitlines()[1:]) # without 1st \n
self.assertTrue(isinstance(results[0], PerformanceTestResult))
r = results[0]
self.assertEquals(r.name, 'Array2D')
self.assertEquals(r.max_rss, 20915200)
def test_parse_results_formatted_text(self):
"""Parse format that Benchmark_Driver prints to console"""
log = ("""
# TEST SAMPLES MIN(μs) MAX(μs) MEAN(μs) SD(μs) MEDIAN(μs) MAX_RSS(B)
3 Array2D 20 2060 2188 2099 0 2099 20915200
Totals 281""")
parser = LogParser()
results = parser.parse_results(log.splitlines()[1:]) # without 1st \n
self.assertTrue(isinstance(results[0], PerformanceTestResult))
r = results[0]
self.assertEquals(r.name, 'Array2D')
self.assertEquals(r.max_rss, 20915200)
def test_parse_results_verbose(self):
"""Parse multiple performance test results with 2 sample formats:
single line for N = 1; two lines for N > 1.
"""
verbose_log = """--- DATA ---
#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
Running AngryPhonebook for 3 samples.
Measuring with scale 78.
Sample 0,11812
Measuring with scale 90.
Sample 1,13898
Sample 2,11467
1,AngryPhonebook,3,11467,13898,12392,1315,11812
Running Array2D for 3 samples.
SetUp 14444
Sample 0,369900
Yielding after ~369918 μs
Sample 1,381039
Yielding after ~381039 μs
Sample 2,371043
3,Array2D,3,369900,381039,373994,6127,371043
Totals,2"""
parser = LogParser()
results = parser.parse_results(verbose_log.split("\n"))
r = results[0]
self.assertEqual(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
("AngryPhonebook", 11467, 13898, 12392, 1315, 11812),
)
self.assertEqual(r.num_samples, r.samples.num_samples)
self.assertEqual(
results[0].samples.all_samples,
[(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)],
)
self.assertEqual(r.yields, None)
r = results[1]
self.assertEqual(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
("Array2D", 369900, 381039, 373994, 6127, 371043),
)
self.assertEqual(r.setup, 14444)
self.assertEqual(r.num_samples, r.samples.num_samples)
self.assertEqual(
results[1].samples.all_samples,
[(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)],
)
yielded = r.yields[0]
self.assertEqual(yielded.before_sample, 1)
self.assertEqual(yielded.after, 369918)
self.assertEqual(r.yields, [(1, 369918), (2, 381039)])
def test_parse_results_verbose(self):
"""Parse multiple performance test results with 2 sample formats:
single line for N = 1; two lines for N > 1.
"""
verbose_log = """--- DATA ---
#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
Running AngryPhonebook for 3 samples.
Measuring with scale 78.
Sample 0,11812
Measuring with scale 90.
Sample 1,13898
Sample 2,11467
1,AngryPhonebook,3,11467,13898,12392,1315,11812
Running Array2D for 3 samples.
SetUp 14444
Sample 0,369900
Yielding after ~369918 μs
Sample 1,381039
Yielding after ~381039 μs
Sample 2,371043
3,Array2D,3,369900,381039,373994,6127,371043
Totals,2"""
parser = LogParser()
results = parser.parse_results(verbose_log.split('\n'))
r = results[0]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('AngryPhonebook', 11467, 13898, 12392, 1315, 11812)
)
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[0].samples.all_samples,
[(0, 78, 11812), (1, 90, 13898), (2, 90, 11467)])
self.assertEquals(r.yields, None)
r = results[1]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('Array2D', 369900, 381039, 373994, 6127, 371043)
)
self.assertEquals(r.setup, 14444)
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[1].samples.all_samples,
[(0, 1, 369900), (1, 1, 381039), (2, 1, 371043)])
yielded = r.yields[0]
self.assertEquals(yielded.before_sample, 1)
self.assertEquals(yielded.after, 369918)
self.assertEquals(r.yields, [(1, 369918), (2, 381039)])
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
7,Array.append.Array.Int?,20,10,10,10,0,10
21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11
42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEqual(results[0].name, "Array.append.Array.Int?")
self.assertEqual(results[1].name, "Bridging.NSArray.as!.Array.NSString")
self.assertEqual(results[2].name, "Flatten.Array.Tuple4.lazy.for-in.Reserve")
def test_parse_environment_verbose(self):
"""Parse stats about environment in verbose mode."""
verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages)
ICS 1338 - 229 = 1109
VCS 2 - 1 = 1
2,AngryPhonebook,3,11269,11884,11657,338,11820
"""
parser = LogParser()
results = parser.parse_results(verbose_log.split('\n'))
r = results[0]
self.assertEquals(r.max_rss, 32768)
self.assertEquals(r.mem_pages, 8)
self.assertEquals(r.voluntary_cs, 1)
self.assertEquals(r.involuntary_cs, 1109)
def test_parse_environment_verbose(self):
"""Parse stats about environment in verbose mode."""
verbose_log = """ MAX_RSS 8937472 - 8904704 = 32768 (8 pages)
ICS 1338 - 229 = 1109
VCS 2 - 1 = 1
2,AngryPhonebook,3,11269,11884,11657,338,11820
"""
parser = LogParser()
results = parser.parse_results(verbose_log.split("\n"))
r = results[0]
self.assertEqual(r.max_rss, 32768)
self.assertEqual(r.mem_pages, 8)
self.assertEqual(r.voluntary_cs, 1)
self.assertEqual(r.involuntary_cs, 1109)
def test_parse_results_csv(self):
"""Ignores uknown lines, extracts data from supported formats."""
log = """#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
7,Array.append.Array.Int?,20,10,10,10,0,10
21,Bridging.NSArray.as!.Array.NSString,20,11,11,11,0,11
42,Flatten.Array.Tuple4.lazy.for-in.Reserve,20,3,4,4,0,4
Total performance tests executed: 1
"""
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'Array.append.Array.Int?')
self.assertEquals(results[1].name,
'Bridging.NSArray.as!.Array.NSString')
self.assertEquals(results[2].name,
'Flatten.Array.Tuple4.lazy.for-in.Reserve')
def test_parse_results_verbose(self):
"""Parse multiple performance test results with 2 sample formats:
single line for N = 1; two lines for N > 1.
"""
verbose_log = """--- DATA ---
#,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us)
Running AngryPhonebook for 3 samples.
Measuring with scale 78.
Sample 0,11812
Measuring with scale 90.
Sample 1,13898
Measuring with scale 91.
Sample 2,11467
1,AngryPhonebook,3,11467,13898,12392,1315,11812
Running Array2D for 3 samples.
Sample 0,369900
Sample 1,381039
Sample 2,371043
3,Array2D,3,369900,381039,373994,6127,371043
Totals,2"""
parser = LogParser()
results = parser.parse_results(verbose_log.split('\n'))
r = results[0]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('AngryPhonebook', 11467, 13898, 12392, 1315, 11812))
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[0].samples.all_samples, [(0, 78, 11812),
(1, 90, 13898),
(2, 91, 11467)])
r = results[1]
self.assertEquals(
(r.name, r.min, r.max, int(r.mean), int(r.sd), r.median),
('Array2D', 369900, 381039, 373994, 6127, 371043))
self.assertEquals(r.num_samples, r.samples.num_samples)
self.assertEquals(results[1].samples.all_samples, [(0, 1, 369900),
(1, 1, 381039),
(2, 1, 371043)])
def test_parse_results_tab_delimited(self):
log = '34\tBitCount\t20\t3\t4\t4\t0\t4'
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEquals(results[0].name, 'BitCount')
def test_parse_results_tab_delimited(self):
log = "34\tBitCount\t20\t3\t4\t4\t0\t4"
parser = LogParser()
results = parser.parse_results(log.splitlines())
self.assertTrue(isinstance(results[0], PerformanceTestResult))
self.assertEqual(results[0].name, "BitCount")
