def test_computes_inter_quartile_range(self): self.assertEqual(self.samples.iqr, 0) self.samples.add(Sample(2, 1, 1025)) self.samples.add(Sample(3, 1, 1050)) self.samples.add(Sample(4, 1, 1075)) self.samples.add(Sample(5, 1, 1100)) self.assertEqual(self.samples.iqr, 50)
def test_init_with_samples(self): self.samples = PerformanceTestSamples( 'B2', [Sample(0, 1, 1000), Sample(1, 1, 1100)]) self.assertEqual(self.samples.count, 2) self.assertEqualtats((self.samples.mean, self.samples.sd, self.samples.range, self.samples.spread), (1050.0, 70.71, 100, 9.52 / 100))
def test_quantile(self): self.assertEqual(self.samples.quantile(1), 1000) self.assertEqual(self.samples.quantile(0), 1000) self.samples.add(Sample(2, 1, 1100)) self.assertEqual(self.samples.quantile(0), 1000) self.assertEqual(self.samples.quantile(1), 1100) self.samples.add(Sample(3, 1, 1050)) self.assertEqual(self.samples.quantile(0), 1000) self.assertEqual(self.samples.quantile(0.5), 1050) self.assertEqual(self.samples.quantile(1), 1100)
def test_computes_five_number_summary(self): self.assertEqualFiveNumberSummary(self.samples, (1000, 1000, 1000, 1000, 1000)) self.samples.add(Sample(2, 1, 1100)) self.assertEqualFiveNumberSummary(self.samples, (1000, 1000, 1000, 1100, 1100)) self.samples.add(Sample(3, 1, 1050)) self.assertEqualFiveNumberSummary(self.samples, (1000, 1000, 1050, 1100, 1100)) self.samples.add(Sample(4, 1, 1025)) self.assertEqualFiveNumberSummary(self.samples, (1000, 1000, 1025, 1050, 1100)) self.samples.add(Sample(5, 1, 1075)) self.assertEqualFiveNumberSummary(self.samples, (1000, 1025, 1050, 1075, 1100))
def test_excludes_outliers_zero_IQR(self): self.samples = PerformanceTestSamples('Tight') self.samples.add(Sample(0, 2, 23)) self.samples.add(Sample(1, 2, 18)) self.samples.add(Sample(2, 2, 18)) self.assertEquals(self.samples.iqr, 0) self.samples.exclude_outliers() self.assertEquals(self.samples.count, 2) self.assertEqualStats((self.samples.min, self.samples.max), (18, 18))
def test_can_handle_zero_runtime(self): # guard against dividing by 0 self.samples = PerformanceTestSamples('Zero') self.samples.add(Sample(0, 1, 0)) self.assertEqualtats( (self.samples.mean, self.samples.sd, self.samples.cv, self.samples.range, self.samples.spread), (0, 0, 0.0, 0, 0.0))
def test_computes_range_spread(self): ss = self.samples self.assertEqualtats( (ss.range, ss.spread), (0, 0)) self.samples.add(Sample(2, 1, 1100)) self.assertEqualtats( (ss.range, ss.spread), (100, 10.0 / 100))
def test_computes_mean_sd_cv(self): ss = self.samples self.assertEqualtats( (ss.mean, ss.sd, ss.cv), (1000.0, 0.0, 0.0)) self.samples.add(Sample(2, 1, 1100)) self.assertEqualtats( (ss.mean, ss.sd, ss.cv), (1050.0, 70.71, 6.7 / 100))
def test_excludes_outliers_top_only(self): ss = [Sample(*map(int, s.split())) for s in '0 1 1, 1 1 2, 2 1 2, 3 1 2, 4 1 3'.split(',')] self.samples = PerformanceTestSamples('Top', ss) self.assertEqualFiveNumberSummary(self.samples, (1, 2, 2, 2, 3)) self.assertEqual(self.samples.iqr, 0) self.samples.exclude_outliers(top_only=True) self.assertEqual(self.samples.count, 4) self.assertEqualtats((self.samples.min, self.samples.max), (1, 2))
def sq(t): # status quo # don't need calibration, but need dummy result for merging samples run(t, s=1) for suffix in list('abcd'): for i in range(0, 20): res = run(t, 1, 0, True) run_name = res.name + ' i0_' + suffix if run_name not in BD.results: BD.results[run_name] = res else: s = res.samples.samples[0] s = Sample(i, s.num_iters, s.runtime) BD.results[run_name].samples.add(s) BD.results[run_name].involuntary_cs += res.involuntary_cs BD.results[run_name].samples.exclude_outliers() print BD.results[run_name].samples
def test_excludes_outliers(self): ss = [ Sample(*map(int, s.split())) for s in "0 1 1000, 1 1 1025, 2 1 1050, 3 1 1075, 4 1 1100, " "5 1 1000, 6 1 1025, 7 1 1050, 8 1 1075, 9 1 1100, " "10 1 1050, 11 1 949, 12 1 1151".split(",") ] self.samples = PerformanceTestSamples("Outliers", ss) self.assertEqual(self.samples.count, 13) self.assertEqualStats((self.samples.mean, self.samples.sd), (1050, 52.36)) self.samples.exclude_outliers() self.assertEqual(self.samples.count, 11) self.assertEqual(self.samples.outliers, ss[11:]) self.assertEqualFiveNumberSummary(self.samples, (1000, 1025, 1050, 1075, 1100)) self.assertEqualStats((self.samples.mean, self.samples.sd), (1050, 35.36))
def load_series(s): """Load series data from JSON dictionary. Returns `PerformanceTestResult` with `PerformanceTestSamples`. """ num_iters = s['num_iters'] ss = PerformanceTestSamples( s['name'], [Sample(i, num_iters, runtime) for (i, runtime) in enumerate(s['data'])]) r = PerformanceTestResult([0, s['name'], ss.count, ss.min, ss.max, ss.mean, ss.sd, ss.median]) r.samples = ss def _set(key): if key in s: setattr(r, key, s[key]) map(_set, ['max_rss', 'involuntary_cs', 'voluntary_cs']) return r
def all_stats(results, setup=0): name = results[0].name.split()[0] + (' all' if not setup else ' fixedAll') ics, vcs = 0, 0 pts = PerformanceTestSamples(name) for result in results: correction = ir(setup / num_iters(result)) samples = (result.samples.samples if not setup else [Sample(s.i, s.num_iters, s.runtime - correction) for s in result.samples.samples]) map(pts.add, samples) ics += result.involuntary_cs vcs += result.voluntary_cs _all = PerformanceTestResult( [0, name, pts.num_samples, pts.min, pts.max, int(pts.mean), int(pts.sd), pts.median]) _all.samples = pts _all.involuntary_cs = ics _all.voluntary_cs = vcs return _all
def setUp(self): self.samples = PerformanceTestSamples("B1") self.samples.add(Sample(7, 42, 1000))
def test_is_iterable(self): s = Sample(1, 2, 3) self.assertEqual(s[0], 1) self.assertEqual(s[1], 2) self.assertEqual(s[2], 3)
def test_has_named_fields(self): s = Sample(1, 2, 3) self.assertEqual(s.i, 1) self.assertEqual(s.num_iters, 2) self.assertEqual(s.runtime, 3)