def print_aggregated_stats(stats_list): """Prints the aggregated stats of given |stats_list|.""" # Skip incomplete stats (probably crashed during this run). We collect # enough runs to make up for an occasional missed run. stat_list = [stats for stats in stats_list if stats.is_complete()] raw_stats = _build_raw_stats(stats_list) # Builds a dict from key to (median, 90-percentile). aggregated_stats = { key: statistics.compute_percentiles(value, (50, 90)) for key, value in raw_stats.iteritems() } # If there is more than 1 stats, print the VPERF= and VRAWPERF= lines. if len(stats_list) > 1: # Print VPERF= lines. for name in _ALL_STAT_VARS: unit = 'ms' if name.endswith('_ms') else 'MB' median, p90 = aggregated_stats[name] print 'VPERF=%(name)s: %(median).2f%(unit)s 90%%=%(p90).2f' % { 'name': name, 'unit': unit, 'median': median, 'p90': p90, } # Print VRAWPERF= line. print 'VRAWPERF=%s' % dict(raw_stats) # Note: since each value is the median for each data set, they are not # guaranteed to add up. print ('\nPERF=boot:%dms (preEmbed:%dms + pluginLoad:%dms + onResume:%dms),' '\n virt:%.1fMB, res:%.1fMB, pdirt:%.1fMB, runs:%d\n' % ( aggregated_stats['boot_time_ms'][0], aggregated_stats['pre_embed_time_ms'][0], aggregated_stats['plugin_load_time_ms'][0], aggregated_stats['on_resume_time_ms'][0], aggregated_stats['app_virt_mem'][0], aggregated_stats['app_res_mem'][0], aggregated_stats['app_pdirt_mem'][0], len(stat_list)))
def print_aggregated_stats(stats_list): """Prints the aggregated stats of given |stats_list|.""" # Skip incomplete stats (probably crashed during this run). We collect # enough runs to make up for an occasional missed run. stat_list = [stats for stats in stats_list if stats.is_complete()] raw_stats = _build_raw_stats(stats_list) # Builds a dict from key to (median, 90-percentile). aggregated_stats = { key: statistics.compute_percentiles(value, (50, 90)) for key, value in raw_stats.iteritems() } # If there is more than 1 stats, print the VPERF= and VRAWPERF= lines. if len(stats_list) > 1: # Print VPERF= lines. for name in _ALL_STAT_VARS: unit = 'ms' if name.endswith('_ms') else 'MB' median, p90 = aggregated_stats[name] print 'VPERF=%(name)s: %(median).2f%(unit)s 90%%=%(p90).2f' % { 'name': name, 'unit': unit, 'median': median, 'p90': p90, } # Print VRAWPERF= line. print 'VRAWPERF=%s' % dict(raw_stats) # Note: since each value is the median for each data set, they are not # guaranteed to add up. print( '\nPERF=boot:%dms (preEmbed:%dms + pluginLoad:%dms + onResume:%dms),' '\n virt:%.1fMB, res:%.1fMB, pdirt:%.1fMB, runs:%d\n' % (aggregated_stats['boot_time_ms'][0], aggregated_stats['pre_embed_time_ms'][0], aggregated_stats['plugin_load_time_ms'][0], aggregated_stats['on_resume_time_ms'][0], aggregated_stats['app_virt_mem'][0], aggregated_stats['app_res_mem'][0], aggregated_stats['app_pdirt_mem'][0], len(stat_list)))
def bootstrap_estimation( ctrl_sample, expt_sample, statistic, confidence_level): """Estimates confidence interval of difference of a statistic by Bootstrap. Args: ctrl_sample: A control sample as a list of numbers. expt_sample: An experiment sample as a list of numbers. statistic: A function that computes a statistic from a sample. confidence_level: An integer that specifies requested confidence level in percentage, e.g. 90, 95, 99. Returns: Estimated range as a number tuple. """ bootstrap_distribution = [] for _ in xrange(1000): bootstrap_distribution.append( statistic(bootstrap_sample(expt_sample)) - statistic(bootstrap_sample(ctrl_sample))) return statistics.compute_percentiles( bootstrap_distribution, (100 - confidence_level, confidence_level))
def bootstrap_estimation(ctrl_sample, expt_sample, statistic, confidence_level): """Estimates confidence interval of difference of a statistic by Bootstrap. Args: ctrl_sample: A control sample as a list of numbers. expt_sample: An experiment sample as a list of numbers. statistic: A function that computes a statistic from a sample. confidence_level: An integer that specifies requested confidence level in percentage, e.g. 90, 95, 99. Returns: Estimated range as a number tuple. """ bootstrap_distribution = [] for _ in xrange(1000): bootstrap_distribution.append( statistic(bootstrap_sample(expt_sample)) - statistic(bootstrap_sample(ctrl_sample))) return statistics.compute_percentiles( bootstrap_distribution, (100 - confidence_level, confidence_level))
def test_compute_percentiles(self): self.assertTrue(all(map(math.isnan, statistics.compute_percentiles([])))) self.assertEquals(statistics.compute_median([5, 8]), statistics.compute_percentiles([5, 8], [50])[0]) self.assertEquals(statistics.compute_median([2, 3, 7]), statistics.compute_percentiles([2, 3, 7], [50])[0]) # All expected values below agree with Google Docs. self.assertEquals((42, 42), statistics.compute_percentiles([42])) self.assertEquals((6.5, 7.7), statistics.compute_percentiles([5, 8])) self.assertEquals((3, 6.2), statistics.compute_percentiles([2, 3, 7])) self.assertEquals((5, 8.4), statistics.compute_percentiles([2, 3, 7, 9])) self.assertEquals((7, 63.6), statistics.compute_percentiles([2, 3, 7, 9, 100])) self.assertEquals((39.5, 43.4), statistics.compute_percentiles([6, 7, 15, 36, 39, 40, 41, 42, 43, 47])) self.assertEquals((40.0, 47.0), statistics.compute_percentiles([6, 7, 15, 36, 39, 40, 41, 42, 43, 47, 49])) self.assertEquals((37.5, 40.5), statistics.compute_percentiles([7, 15, 36, 39, 40, 41])) self.assertEquals((39, 42.2), statistics.compute_percentiles([6, 7, 15, 36, 39, 40, 41, 42, 43]))
def test_compute_percentiles(self): self.assertTrue( all(map(math.isnan, statistics.compute_percentiles([])))) self.assertEquals(statistics.compute_median([5, 8]), statistics.compute_percentiles([5, 8], [50])[0]) self.assertEquals(statistics.compute_median([2, 3, 7]), statistics.compute_percentiles([2, 3, 7], [50])[0]) # All expected values below agree with Google Docs. self.assertEquals((42, 42), statistics.compute_percentiles([42])) self.assertEquals((6.5, 7.7), statistics.compute_percentiles([5, 8])) self.assertEquals((3, 6.2), statistics.compute_percentiles([2, 3, 7])) self.assertEquals((5, 8.4), statistics.compute_percentiles([2, 3, 7, 9])) self.assertEquals((7, 63.6), statistics.compute_percentiles([2, 3, 7, 9, 100])) self.assertEquals((39.5, 43.4), statistics.compute_percentiles( [6, 7, 15, 36, 39, 40, 41, 42, 43, 47])) self.assertEquals((40.0, 47.0), statistics.compute_percentiles( [6, 7, 15, 36, 39, 40, 41, 42, 43, 47, 49])) self.assertEquals( (37.5, 40.5), statistics.compute_percentiles([7, 15, 36, 39, 40, 41])) self.assertEquals( (39, 42.2), statistics.compute_percentiles([6, 7, 15, 36, 39, 40, 41, 42, 43]))