def format_result_value(bench): loops = None warmups = None for run in bench._runs: if run._is_calibration_warmups(): warmups = run._get_calibration_warmups() elif run._is_recalibration_warmups(): warmups = run._get_calibration_warmups() elif run._is_recalibration_loops(): loops = run._get_calibration_loops() elif run._is_calibration_warmups(): loops = run._get_calibration_loops() elif run._is_calibration_loops(): loops = run._get_calibration_loops() else: loops = None warmups = None break info = [] if loops is not None: info.append(format_number(loops, 'loop')) if warmups is not None: info.append(format_number(warmups, 'warmup')) if info: return '<calibration: %s>' % ', '.join(info) return _format_result_value(bench)
def format_result(bench, prefix=True): loops = None warmups = None for run in bench._runs: if run._is_calibration_warmups(): warmups = run._get_calibration_warmups() elif run._is_recalibration_warmups(): warmups = run._get_calibration_warmups() elif run._is_recalibration_loops(): loops = run._get_calibration_loops() elif run._is_calibration_warmups(): loops = run._get_calibration_loops() elif run._is_calibration_loops(): loops = run._get_calibration_loops() else: loops = None warmups = None break info = [] if loops is not None: info.append(format_number(loops, 'loop')) if warmups is not None: info.append(format_number(warmups, 'warmup')) if info: return 'Calibration: %s' % ', '.join(info) text = _format_result_value(bench) if bench.get_nvalue() >= 2: return 'Mean +- std dev: %s' % text else: return text
def calibration_done(self): if self.args.verbose: print("Calibration: %s, %s" % (format_number(self.args.warmups, 'warmup'), format_number(self.args.loops, 'loop'))) self.calibrate_loops = 0 self.calibrate_warmups = 0
def _compute_values(self, values, nvalue, is_warmup=False, calibrate_loops=False, start=0): unit = self.metadata.get('unit') args = self.args if nvalue < 1: raise ValueError("nvalue must be >= 1") if self.loops <= 0: raise ValueError("loops must be >= 1") if is_warmup: value_name = 'Warmup' else: value_name = 'Value' index = 1 inner_loops = self.inner_loops if not inner_loops: inner_loops = 1 while True: if index > nvalue: break raw_value = self.task_func(self, self.loops) raw_value = float(raw_value) value = raw_value / (self.loops * inner_loops) if not value and not calibrate_loops: raise ValueError("benchmark function returned zero") if is_warmup: values.append((self.loops, value)) else: values.append(value) if args.verbose: text = format_value(unit, value) if is_warmup: text = ('%s (loops: %s, raw: %s)' % (text, format_number(self.loops), format_value(unit, raw_value))) print("%s %s: %s" % (value_name, start + index, text)) if calibrate_loops and raw_value < args.min_time: if self.loops * 2 > MAX_LOOPS: print("ERROR: failed to calibrate the number of loops") print("Raw timing %s with %s is still smaller than " "the minimum time of %s" % (format_value(unit, raw_value), format_number(self.loops, 'loop'), format_timedelta(args.min_time))) sys.exit(1) self.loops *= 2 # need more values for the calibration nvalue += 1 index += 1
def _spawn_workers(self, python=None, newline=True): bench = None args = self.args verbose = args.verbose quiet = args.quiet nprocess = args.processes old_loops = self.args.loops need_calibration = (not args.loops) if need_calibration: nprocess += 1 calibrate = need_calibration if verbose and self._worker_task > 0: print() for process in range(1, nprocess + 1): suite = self._spawn_worker(python, calibrate) if suite is None: raise RuntimeError("perf worker process didn't produce JSON result") benchmarks = suite.get_benchmarks() if len(benchmarks) != 1: raise ValueError("worker produced %s benchmarks instead of 1" % len(benchmarks)) worker_bench = benchmarks[0] if verbose: run = worker_bench.get_runs()[-1] run_index = '%s/%s' % (process, nprocess) for line in format_run(worker_bench, run_index, run): print(line) elif not quiet: print(".", end='') if calibrate: # Use the first worker to calibrate the benchmark. Use a worker # process rather than the main process because worker is a # little bit more isolated and so should be more reliable. first_run = worker_bench.get_runs()[0] args.loops = first_run._get_loops() if verbose: print("Calibration: use %s loops" % format_number(args.loops)) calibrate = False if bench is not None: bench.add_runs(worker_bench) else: bench = worker_bench sys.stdout.flush() if not quiet and newline: print() # restore the old value of loops, to recalibrate for the next # benchmark function if loops=0 args.loops = old_loops return bench
def format_result(bench, prefix=True): loops = bench._only_calibration() if loops is not None: return 'Calibration: %s' % format_number(loops, 'loop') text = format_result_value(bench) if bench.get_nvalue() >= 2: return 'Mean +- std dev: %s' % text else: return text
def format_result_value(bench): loops = bench._only_calibration() if loops is not None: return '<calibration: %s>' % format_number(loops, 'loop') mean = bench.mean() if bench.get_nvalue() >= 2: args = bench.format_values((mean, bench.stdev())) return '%s +- %s' % args else: return bench.format_value(mean)
def calibrate_warmups(self): # calibrate the number of warmups if self.loops < 1: raise ValueError("loops must be >= 1") if self.args.recalibrate_warmups: nwarmup = self.args.warmups else: nwarmup = 1 unit = self.metadata.get('unit') start = 0 # test_calibrate_warmups() requires at least 2 values per sample while True: total = nwarmup + WARMUP_SAMPLE_SIZE * 2 nvalue = total - len(self.warmups) if nvalue: self._compute_values(self.warmups, nvalue, is_warmup=True, start=start) start += nvalue if self.test_calibrate_warmups(nwarmup, unit): break if len(self.warmups) >= MAX_WARMUP_VALUES: print("ERROR: failed to calibrate the number of warmups") values = [format_value(unit, value) for loops, value in self.warmups] print("Values (%s): %s" % (len(values), ', '.join(values))) sys.exit(1) nwarmup += 1 if self.args.verbose: print("Calibration: use %s warmups" % format_number(nwarmup)) print() if self.args.recalibrate_warmups: self.metadata['recalibrate_warmups'] = nwarmup else: self.metadata['calibrate_warmups'] = nwarmup
def calibrate_loops(self): args = self.args if not args.recalibrate_loops: self.loops = 1 if args.warmups is not None: nvalue = args.warmups else: nvalue = 1 nvalue += args.values self._compute_values(self.warmups, nvalue, is_warmup=True, calibrate_loops=True) if args.verbose: print() print("Calibration: use %s loops" % format_number(self.loops)) print() if args.recalibrate_loops: self.metadata['recalibrate_loops'] = self.loops else: self.metadata['calibrate_loops'] = self.loops
def format_run(bench, run_index, run, common_metadata=None, raw=False, verbose=0, lines=None): if lines is None: lines = [] inner_loops = run.get_inner_loops() if run._is_calibration(): if run._is_calibration_warmups(): warmups = run._get_calibration_warmups() action = 'calibrate the number of warmups: %s' % format_number( warmups) elif run._is_recalibration_warmups(): warmups = run._get_calibration_warmups() action = 'recalibrate the number of warmups: %s' % format_number( warmups) elif run._is_recalibration_loops(): loops = run._get_calibration_loops() action = 'recalibrate the number of loops: %s' % format_number( loops) else: loops = run._get_calibration_loops() action = 'calibrate the number of loops: %s' % format_number(loops) lines.append("Run %s: %s" % (run_index, action)) if raw: name = 'raw calibrate' else: name = 'calibrate' for index, warmup in enumerate(run.warmups, 1): loops, value = warmup raw_value = value * (loops * inner_loops) if raw: text = format_timedelta(raw_value) text = ("%s (loops: %s)" % (format_timedelta(raw_value), format_number(loops))) else: text = ("%s (loops: %s, raw: %s)" % (format_timedelta(value), format_number(loops), format_timedelta(raw_value))) lines.append("- %s %s: %s" % (name, index, text)) else: show_warmup = (verbose >= 0) total_loops = run.get_total_loops() values = run.values if raw: warmups = [ bench.format_value(value * (loops * inner_loops)) for loops, value in run.warmups ] values = [value * total_loops for value in values] else: warmups = run.warmups if warmups: warmups = [value for loops, value in warmups] warmups = _format_values_diff(bench, warmups, raw, total_loops) values = _format_values_diff(bench, values, raw, total_loops) if verbose >= 0: loops = run.get_loops() lines.append("Run %s: %s, %s, %s" % (run_index, format_number(len(warmups), 'warmup'), format_number(len(values), 'value'), format_number(loops, 'loop'))) else: lines.append("Run %s:" % run_index) if warmups and show_warmup: if raw: name = 'raw warmup' else: name = 'warmup' for index, warmup in enumerate(warmups, 1): lines.append('- %s %s: %s' % (name, index, warmup)) if raw: name = 'raw value' else: name = 'value' for index, value in enumerate(values, 1): lines.append('- %s %s: %s' % (name, index, value)) if verbose > 0: metadata = run.get_metadata() if metadata: lines.append('- Metadata:') for name, value in sorted(metadata.items()): if common_metadata and name in common_metadata: continue value = _format_metadata(name, value) lines.append(' %s: %s' % (name, value)) return lines
def format_stats(bench, lines): fmt = bench.format_value values = bench.get_values() nrun = bench.get_nrun() nvalue = len(values) empty_line(lines) # Total duration duration = bench.get_total_duration() if duration: lines.append("Total duration: %s" % format_seconds(duration)) # Start/End dates dates = bench.get_dates() if dates: start, end = dates lines.append("Start date: %s" % format_datetime(start, microsecond=False)) lines.append("End date: %s" % format_datetime(end, microsecond=False)) # Raw value minimize/maximum raw_values = bench._get_raw_values() lines.append("Raw value minimum: %s" % bench.format_value(min(raw_values))) lines.append("Raw value maximum: %s" % bench.format_value(max(raw_values))) lines.append('') # Number of values ncalibration_runs = sum(run._is_calibration() for run in bench._runs) lines.append("Number of calibration run: %s" % format_number(ncalibration_runs)) lines.append("Number of run with values: %s" % (format_number(nrun - ncalibration_runs))) lines.append("Total number of run: %s" % format_number(nrun)) lines.append('') # Number of values nwarmup = bench._get_nwarmup() text = format_number(nwarmup) if isinstance(nwarmup, float): text += ' (average)' lines.append('Number of warmup per run: %s' % text) nvalue_per_run = bench._get_nvalue_per_run() text = format_number(nvalue_per_run) if isinstance(nvalue_per_run, float): text += ' (average)' lines.append('Number of value per run: %s' % text) # Loop iterations per value loops = bench.get_loops() inner_loops = bench.get_inner_loops() total_loops = loops * inner_loops if isinstance(total_loops, int): text = format_number(total_loops) else: text = "%s (average)" % total_loops if not(isinstance(inner_loops, int) and inner_loops == 1): if isinstance(loops, int): loops = format_number(loops, 'outer-loop') else: loops = '%.1f outer-loops (average)' if isinstance(inner_loops, int): inner_loops = format_number(inner_loops, 'inner-loop') else: inner_loops = "%.1f inner-loops (average)" % inner_loops text = '%s (%s x %s)' % (text, loops, inner_loops) lines.append("Loop iterations per value: %s" % text) lines.append("Total number of values: %s" % format_number(nvalue)) lines.append('') # Minimum table = [] table.append(("Minimum", bench.format_value(min(values)))) # Median +- MAD median = bench.median() if len(values) > 2: median_abs_dev = bench.median_abs_dev() table.append(("Median +- MAD", "%s +- %s" % bench.format_values((median, median_abs_dev)))) else: table.append(("Mean", bench.format_value(median))) # Mean +- std dev mean = bench.mean() if len(values) > 2: stdev = bench.stdev() table.append(("Mean +- std dev", "%s +- %s" % bench.format_values((mean, stdev)))) else: table.append(("Mean", bench.format_value(mean))) table.append(("Maximum", bench.format_value(max(values)))) # Render table width = max(len(row[0]) + 1 for row in table) for key, value in table: key = (key + ':').ljust(width) lines.append("%s %s" % (key, value)) lines.append('') def format_limit(mean, value): return ("%s (%+.0f%% of the mean)" % (fmt(value), (value - mean) * 100.0 / mean)) # Percentiles for p in (0, 5, 25, 50, 75, 95, 100): text = format_limit(mean, bench.percentile(p)) text = "%3sth percentile: %s" % (p, text) name = PERCENTILE_NAMES.get(p) if name: text = '%s -- %s' % (text, name) lines.append(text) lines.append('') # Outliers q1 = bench.percentile(25) q3 = bench.percentile(75) iqr = q3 - q1 outlier_min = (q1 - 1.5 * iqr) outlier_max = (q3 + 1.5 * iqr) noutlier = sum(not(outlier_min <= value <= outlier_max) for value in values) bounds = bench.format_values((outlier_min, outlier_max)) lines.append('Number of outlier (out of %s..%s): %s' % (bounds[0], bounds[1], format_number(noutlier))) return lines
def format_run(bench, run_index, run, common_metadata=None, raw=False, verbose=0, lines=None): if lines is None: lines = [] inner_loops = run.get_inner_loops() if run._is_calibration(): if run._is_calibration_warmups(): warmups = run._get_calibration_warmups() action = 'calibrate the number of warmups: %s' % format_number(warmups) elif run._is_recalibration_warmups(): warmups = run._get_calibration_warmups() action = 'recalibrate the number of warmups: %s' % format_number(warmups) elif run._is_recalibration_loops(): loops = run._get_calibration_loops() action = 'recalibrate the number of loops: %s' % format_number(loops) else: loops = run._get_calibration_loops() action = 'calibrate the number of loops: %s' % format_number(loops) lines.append("Run %s: %s" % (run_index, action)) if raw: name = 'raw calibrate' else: name = 'calibrate' unit = bench.get_unit() format_value = bench.format_value for index, warmup in enumerate(run.warmups, 1): loops, value = warmup raw_value = value * (loops * inner_loops) if raw: text = format_value(raw_value) text = ("%s (loops: %s)" % (format_value(raw_value), format_number(loops))) # when using --track-memory, displaying value * loops doesn't make # sense, so only display raw value if the unit is seconds elif unit == 'second': text = ("%s (loops: %s, raw: %s)" % (format_value(value), format_number(loops), format_value(raw_value))) else: text = ("%s (loops: %s)" % (format_value(value), format_number(loops))) lines.append("- %s %s: %s" % (name, index, text)) else: show_warmup = (verbose >= 0) total_loops = run.get_total_loops() values = run.values if raw: warmups = [bench.format_value(value * (loops * inner_loops)) for loops, value in run.warmups] values = [value * total_loops for value in values] else: warmups = run.warmups if warmups: warmups = [value for loops, value in warmups] warmups = _format_values_diff(bench, warmups, raw, total_loops) values = _format_values_diff(bench, values, raw, total_loops) if verbose >= 0: loops = run.get_loops() lines.append("Run %s: %s, %s, %s" % (run_index, format_number(len(warmups), 'warmup'), format_number(len(values), 'value'), format_number(loops, 'loop'))) else: lines.append("Run %s:" % run_index) if warmups and show_warmup: if raw: name = 'raw warmup' else: name = 'warmup' for index, warmup in enumerate(warmups, 1): lines.append('- %s %s: %s' % (name, index, warmup)) if raw: name = 'raw value' else: name = 'value' for index, value in enumerate(values, 1): lines.append('- %s %s: %s' % (name, index, value)) if verbose > 0: metadata = run.get_metadata() if metadata: lines.append('- Metadata:') for name, value in sorted(metadata.items()): if common_metadata and name in common_metadata: continue value = _format_metadata(name, value) lines.append(' %s: %s' % (name, value)) return lines
def format_stats(bench, lines): fmt = bench.format_value values = bench.get_values() nrun = bench.get_nrun() nvalue = len(values) empty_line(lines) # Total duration duration = bench.get_total_duration() if duration: lines.append("Total duration: %s" % format_seconds(duration)) # Start/End dates dates = bench.get_dates() if dates: start, end = dates lines.append("Start date: %s" % format_datetime(start, microsecond=False)) lines.append("End date: %s" % format_datetime(end, microsecond=False)) # Raw value minimize/maximum raw_values = bench._get_raw_values() lines.append("Raw value minimum: %s" % bench.format_value(min(raw_values))) lines.append("Raw value maximum: %s" % bench.format_value(max(raw_values))) lines.append('') # Number of values ncalibration_runs = sum(run._is_calibration() for run in bench._runs) lines.append("Number of calibration run: %s" % format_number(ncalibration_runs)) lines.append("Number of run with values: %s" % (format_number(nrun - ncalibration_runs))) lines.append("Total number of run: %s" % format_number(nrun)) lines.append('') # Number of values nwarmup = bench._get_nwarmup() text = format_number(nwarmup) if isinstance(nwarmup, float): text += ' (average)' lines.append('Number of warmup per run: %s' % text) nvalue_per_run = bench._get_nvalue_per_run() text = format_number(nvalue_per_run) if isinstance(nvalue_per_run, float): text += ' (average)' lines.append('Number of value per run: %s' % text) # Loop iterations per value loops = bench.get_loops() inner_loops = bench.get_inner_loops() total_loops = loops * inner_loops if isinstance(total_loops, int): text = format_number(total_loops) else: text = "%s (average)" % total_loops if not (isinstance(inner_loops, int) and inner_loops == 1): if isinstance(loops, int): loops = format_number(loops, 'outer-loop') else: loops = '%.1f outer-loops (average)' if isinstance(inner_loops, int): inner_loops = format_number(inner_loops, 'inner-loop') else: inner_loops = "%.1f inner-loops (average)" % inner_loops text = '%s (%s x %s)' % (text, loops, inner_loops) lines.append("Loop iterations per value: %s" % text) lines.append("Total number of values: %s" % format_number(nvalue)) lines.append('') # Minimum table = [] table.append(("Minimum", bench.format_value(min(values)))) # Median +- MAD median = bench.median() if len(values) > 2: median_abs_dev = bench.median_abs_dev() table.append(("Median +- MAD", "%s +- %s" % bench.format_values( (median, median_abs_dev)))) else: table.append(("Mean", bench.format_value(median))) # Mean +- std dev mean = bench.mean() if len(values) > 2: stdev = bench.stdev() table.append(("Mean +- std dev", "%s +- %s" % bench.format_values( (mean, stdev)))) else: table.append(("Mean", bench.format_value(mean))) table.append(("Maximum", bench.format_value(max(values)))) # Render table width = max(len(row[0]) + 1 for row in table) for key, value in table: key = (key + ':').ljust(width) lines.append("%s %s" % (key, value)) lines.append('') def format_limit(mean, value): return ("%s (%+.0f%% of the mean)" % (fmt(value), (value - mean) * 100.0 / mean)) # Percentiles for p in (0, 5, 25, 50, 75, 95, 100): text = format_limit(mean, bench.percentile(p)) text = "%3sth percentile: %s" % (p, text) name = PERCENTILE_NAMES.get(p) if name: text = '%s -- %s' % (text, name) lines.append(text) lines.append('') # Outliers q1 = bench.percentile(25) q3 = bench.percentile(75) iqr = q3 - q1 outlier_min = (q1 - 1.5 * iqr) outlier_max = (q3 + 1.5 * iqr) noutlier = sum(not (outlier_min <= value <= outlier_max) for value in values) bounds = bench.format_values((outlier_min, outlier_max)) lines.append('Number of outlier (out of %s..%s): %s' % (bounds[0], bounds[1], format_number(noutlier))) return lines
def test_calibrate_warmups(self, nwarmup, unit): half = nwarmup + (len(self.warmups) - nwarmup) // 2 sample1 = [value for loops, value in self.warmups[nwarmup:half]] sample2 = [value for loops, value in self.warmups[half:]] first_value = sample1[0] # test if the first value is an outlier values = sample1[1:] + sample2 q1 = percentile(values, 0.25) q3 = percentile(values, 0.75) iqr = q3 - q1 outlier_max = (q3 + 1.5 * iqr) # only check maximum, not minimum outlier = not(first_value <= outlier_max) mean1 = statistics.mean(sample1) mean2 = statistics.mean(sample2) mean_diff = (mean1 - mean2) / float(mean2) s1_q1 = percentile(sample1, 0.25) s2_q1 = percentile(sample2, 0.25) s1_q3 = percentile(sample1, 0.75) s2_q3 = percentile(sample2, 0.75) q1_diff = (s1_q1 - s2_q1) / float(s2_q1) q3_diff = (s1_q3 - s2_q3) / float(s2_q3) mad1 = median_abs_dev(sample1) mad2 = median_abs_dev(sample2) # FIXME: handle division by zero mad_diff = (mad1 - mad2) / float(mad2) if self.args.verbose: stdev1 = statistics.stdev(sample1) stdev2 = statistics.stdev(sample2) stdev_diff = (stdev1 - stdev2) / float(stdev2) sample1_str = format_values(unit, (s1_q1, mean1, s1_q3, stdev1, mad1)) sample2_str = format_values(unit, (s2_q1, mean2, s2_q3, stdev2, mad2)) print("Calibration: warmups=%s" % format_number(nwarmup)) print(" first value: %s, outlier? %s (max: %s)" % (format_value(unit, first_value), outlier, format_value(unit, outlier_max))) print(" sample1(%s): Q1=%s mean=%s Q3=%s stdev=%s MAD=%s" % (len(sample1), sample1_str[0], sample1_str[1], sample1_str[2], sample1_str[3], sample1_str[4])) print(" sample2(%s): Q1=%s mean=%s Q3=%s stdev=%s MAD=%s" % (len(sample2), sample2_str[0], sample2_str[1], sample2_str[2], sample2_str[3], sample2_str[4])) print(" diff: Q1=%+.0f%% mean=%+.0f%% Q3=%+.0f%% stdev=%+.0f%% MAD=%+.0f%%" % (q1_diff * 100, mean_diff * 100, q3_diff * 100, stdev_diff * 100, mad_diff * 100)) if outlier: return False if not(-0.5 <= mean_diff <= 0.10): return False if abs(mad_diff) > 0.10: return False if abs(q1_diff) > 0.05: return False if abs(q3_diff) > 0.05: return False return True
def run_bench(self, nvalue, is_warmup=False, is_calibrate=False, calibrate=False): unit = self.metadata.get('unit') args = self.args if self.loops <= 0: raise ValueError("loops must be >= 1") if is_calibrate: value_name = 'Calibration' elif is_warmup: value_name = 'Warmup' else: value_name = 'Value' values = [] index = 1 inner_loops = self.inner_loops if not inner_loops: inner_loops = 1 while True: if index > nvalue: break raw_value = self.task_func(self, self.loops) raw_value = float(raw_value) value = raw_value / (self.loops * inner_loops) if not value and not (is_calibrate or is_warmup): raise ValueError("benchmark function returned zero") if is_warmup: values.append((self.loops, value)) else: values.append(value) if args.verbose: text = format_value(unit, value) if is_warmup or is_calibrate: text = ('%s (%s: %s)' % (text, format_number(self.loops, 'loop'), format_value(unit, raw_value))) print("%s %s: %s" % (value_name, index, text)) if calibrate and raw_value < args.min_time: self.loops *= 2 if self.loops > MAX_LOOPS: raise ValueError("error in calibration, loops is " "too big: %s" % self.loops) # need more values for the calibration nvalue += 1 index += 1 if args.verbose: if is_calibrate: print("Calibration: use %s loops" % format_number(self.loops)) print() return values
def test_format_number(self): # plural self.assertEqual(format_number(0, 'unit'), '0 units') self.assertEqual(format_number(1, 'unit'), '1 unit') self.assertEqual(format_number(2, 'unit'), '2 units') self.assertEqual(format_number(123, 'unit'), '123 units') # powers of 10 self.assertEqual(format_number(10**3, 'unit'), '1000 units') self.assertEqual(format_number(10**4, 'unit'), '10^4 units') self.assertEqual(format_number(10**4 + 1, 'unit'), '10001 units') self.assertEqual(format_number(33 * 10**4, 'unit'), '330000 units') # powers of 10 self.assertEqual(format_number(2**10, 'unit'), '1024 units') self.assertEqual(format_number(2**15, 'unit'), '2^15 units') self.assertEqual(format_number(2**15), '2^15') self.assertEqual(format_number(2**10 + 1, 'unit'), '1025 units')
def test_format_number(self): # plural self.assertEqual(format_number(0, 'unit'), '0 units') self.assertEqual(format_number(1, 'unit'), '1 unit') self.assertEqual(format_number(2, 'unit'), '2 units') self.assertEqual(format_number(123, 'unit'), '123 units') # powers of 10 self.assertEqual(format_number(10 ** 3, 'unit'), '1000 units') self.assertEqual(format_number(10 ** 4, 'unit'), '10^4 units') self.assertEqual(format_number(10 ** 4 + 1, 'unit'), '10001 units') self.assertEqual(format_number(33 * 10 ** 4, 'unit'), '330000 units') # powers of 10 self.assertEqual(format_number(2 ** 10, 'unit'), '1024 units') self.assertEqual(format_number(2 ** 15, 'unit'), '2^15 units') self.assertEqual(format_number(2 ** 15), '2^15') self.assertEqual(format_number(2 ** 10 + 1, 'unit'), '1025 units')
def format_run(bench, run_index, run, common_metadata=None, raw=False, verbose=0, lines=None): if lines is None: lines = [] inner_loops = run._get_inner_loops() if run._is_calibration(): lines.append("Run %s: calibrate" % (run_index, )) for loops, value in run.warmups: raw_value = value * (loops * inner_loops) if raw: lines.append("- %s: %s" % (format_number( loops, 'loop'), format_timedelta(raw_value))) else: lines.append( "- %s: %s (raw: %s)" % (format_number(loops, 'loop'), format_timedelta(value), format_timedelta(raw_value))) return lines show_warmup = (verbose >= 0) total_loops = run.get_total_loops() def format_values(values): values_str = [bench.format_value(value) for value in values] mean = bench.mean() max_delta = mean * 0.05 for index, value in enumerate(values): if raw: value = float(value) / total_loops delta = float(value) - mean if abs(delta) > max_delta: values_str[index] += ' (%+.0f%%)' % (delta * 100 / mean) return values_str values = run.values if raw: warmups = [ bench.format_value(value * (loops * inner_loops)) for loops, value in run.warmups ] values = [value * total_loops for value in values] else: warmups = run.warmups if warmups: warmups = [value for loops, value in warmups] warmups = format_values(warmups) values = format_values(values) if raw: name = 'raw values' else: name = 'values' text = '%s (%s): %s' % (name, len(values), ', '.join(values)) if warmups and show_warmup: if raw: name = 'raw warmups' else: name = 'warmups' text = ('%s (%s): %s; %s' % (name, len(warmups), ', '.join(warmups), text)) text = "Run %s: %s" % (run_index, text) lines.append(text) if verbose > 0: prefix = ' ' metadata = run.get_metadata() for name, value in sorted(metadata.items()): if common_metadata and name in common_metadata: continue value = _format_metadata(name, value) lines.append('%s%s: %s' % (prefix, name, value)) return lines