def format_result(bench, prefix=True):
loops = None
warmups = None
for run in bench._runs:
if run._is_calibration_warmups():
warmups = run._get_calibration_warmups()
elif run._is_recalibration_warmups():
warmups = run._get_calibration_warmups()
elif run._is_recalibration_loops():
loops = run._get_calibration_loops()
elif run._is_calibration_warmups():
loops = run._get_calibration_loops()
elif run._is_calibration_loops():
loops = run._get_calibration_loops()
else:
loops = None
warmups = None
break
info = []
if loops is not None:
info.append(format_number(loops, 'loop'))
if warmups is not None:
info.append(format_number(warmups, 'warmup'))
if info:
return 'Calibration: %s' % ', '.join(info)
text = _format_result_value(bench)
if bench.get_nvalue() >= 2:
return 'Mean +- std dev: %s' % text
else:
return text
def calibration_done(self):
if self.args.verbose:
print("Calibration: %s, %s" %
(format_number(self.args.warmups, 'warmup'),
format_number(self.args.loops, 'loop')))
self.calibrate_loops = 0
self.calibrate_warmups = 0
def format_result_value(bench):
loops = None
warmups = None
for run in bench._runs:
if run._is_calibration_warmups():
warmups = run._get_calibration_warmups()
elif run._is_recalibration_warmups():
warmups = run._get_calibration_warmups()
elif run._is_recalibration_loops():
loops = run._get_calibration_loops()
elif run._is_calibration_warmups():
loops = run._get_calibration_loops()
elif run._is_calibration_loops():
loops = run._get_calibration_loops()
else:
loops = None
warmups = None
break
info = []
if loops is not None:
info.append(format_number(loops, 'loop'))
if warmups is not None:
info.append(format_number(warmups, 'warmup'))
if info:
return '<calibration: %s>' % ', '.join(info)
return _format_result_value(bench)
def _compute_values(self,
values,
nvalue,
is_warmup=False,
calibrate_loops=False,
start=0):
unit = self.metadata.get('unit')
args = self.args
if nvalue < 1:
raise ValueError("nvalue must be >= 1")
if self.loops <= 0:
raise ValueError("loops must be >= 1")
if is_warmup:
value_name = 'Warmup'
else:
value_name = 'Value'
index = 1
inner_loops = self.inner_loops
if not inner_loops:
inner_loops = 1
while True:
if index > nvalue:
break
raw_value = self.task_func(self, self.loops)
raw_value = float(raw_value)
value = raw_value / (self.loops * inner_loops)
if not value and not calibrate_loops:
raise ValueError("benchmark function returned zero")
if is_warmup:
values.append((self.loops, value))
else:
values.append(value)
if args.verbose:
text = format_value(unit, value)
if is_warmup:
text = ('%s (loops: %s, raw: %s)' %
(text, format_number(
self.loops), format_value(unit, raw_value)))
print("%s %s: %s" % (value_name, start + index, text))
if calibrate_loops and raw_value < args.min_time:
if self.loops * 2 > MAX_LOOPS:
print("ERROR: failed to calibrate the number of loops")
print("Raw timing %s with %s is still smaller than "
"the minimum time of %s" %
(format_value(unit, raw_value),
format_number(self.loops, 'loop'),
format_timedelta(args.min_time)))
sys.exit(1)
self.loops *= 2
# need more values for the calibration
nvalue += 1
index += 1
def calibrate_warmups(self):
# calibrate the number of warmups
if self.loops < 1:
raise ValueError("loops must be >= 1")
if self.args.recalibrate_warmups:
nwarmup = self.args.warmups
else:
nwarmup = 1
unit = self.metadata.get('unit')
start = 0
# test_calibrate_warmups() requires at least 2 values per sample
while True:
total = nwarmup + WARMUP_SAMPLE_SIZE * 2
nvalue = total - len(self.warmups)
if nvalue:
self._compute_values(self.warmups,
nvalue,
is_warmup=True,
start=start)
start += nvalue
if self.test_calibrate_warmups(nwarmup, unit):
break
if len(self.warmups) >= MAX_WARMUP_VALUES:
print("ERROR: failed to calibrate the number of warmups")
values = [
format_value(unit, value) for loops, value in self.warmups
]
print("Values (%s): %s" % (len(values), ', '.join(values)))
sys.exit(1)
nwarmup += 1
if self.args.verbose:
print("Calibration: use %s warmups" % format_number(nwarmup))
print()
if self.args.recalibrate_warmups:
self.metadata['recalibrate_warmups'] = nwarmup
else:
self.metadata['calibrate_warmups'] = nwarmup
def calibrate_loops(self):
args = self.args
if not args.recalibrate_loops:
self.loops = 1
if args.warmups is not None:
nvalue = args.warmups
else:
nvalue = 1
nvalue += args.values
self._compute_values(self.warmups, nvalue,
is_warmup=True,
calibrate_loops=True)
if args.verbose:
print()
print("Calibration: use %s loops" % format_number(self.loops))
print()
if args.recalibrate_loops:
self.metadata['recalibrate_loops'] = self.loops
else:
self.metadata['calibrate_loops'] = self.loops
def test_format_number(self):
# plural
self.assertEqual(format_number(0, 'unit'), '0 units')
self.assertEqual(format_number(1, 'unit'), '1 unit')
self.assertEqual(format_number(2, 'unit'), '2 units')
self.assertEqual(format_number(123, 'unit'), '123 units')
# powers of 10
self.assertEqual(format_number(10 ** 3, 'unit'),
'1000 units')
self.assertEqual(format_number(10 ** 4, 'unit'),
'10^4 units')
self.assertEqual(format_number(10 ** 4 + 1, 'unit'),
'10001 units')
self.assertEqual(format_number(33 * 10 ** 4, 'unit'),
'330000 units')
# powers of 10
self.assertEqual(format_number(2 ** 10, 'unit'),
'1024 units')
self.assertEqual(format_number(2 ** 15, 'unit'),
'2^15 units')
self.assertEqual(format_number(2 ** 15),
'2^15')
self.assertEqual(format_number(2 ** 10 + 1, 'unit'),
'1025 units')
def test_calibrate_warmups(self, nwarmup, unit):
half = nwarmup + (len(self.warmups) - nwarmup) // 2
sample1 = [value for loops, value in self.warmups[nwarmup:half]]
sample2 = [value for loops, value in self.warmups[half:]]
first_value = sample1[0]
# test if the first value is an outlier
values = sample1[1:] + sample2
q1 = percentile(values, 0.25)
q3 = percentile(values, 0.75)
iqr = q3 - q1
outlier_max = (q3 + 1.5 * iqr)
# only check maximum, not minimum
outlier = not(first_value <= outlier_max)
mean1 = statistics.mean(sample1)
mean2 = statistics.mean(sample2)
mean_diff = (mean1 - mean2) / float(mean2)
s1_q1 = percentile(sample1, 0.25)
s2_q1 = percentile(sample2, 0.25)
s1_q3 = percentile(sample1, 0.75)
s2_q3 = percentile(sample2, 0.75)
q1_diff = (s1_q1 - s2_q1) / float(s2_q1)
q3_diff = (s1_q3 - s2_q3) / float(s2_q3)
mad1 = median_abs_dev(sample1)
mad2 = median_abs_dev(sample2)
# FIXME: handle division by zero
mad_diff = (mad1 - mad2) / float(mad2)
if self.args.verbose:
stdev1 = statistics.stdev(sample1)
stdev2 = statistics.stdev(sample2)
stdev_diff = (stdev1 - stdev2) / float(stdev2)
sample1_str = format_values(unit, (s1_q1, mean1, s1_q3, stdev1, mad1))
sample2_str = format_values(unit, (s2_q1, mean2, s2_q3, stdev2, mad2))
print("Calibration: warmups=%s" % format_number(nwarmup))
print(" first value: %s, outlier? %s (max: %s)"
% (format_value(unit, first_value), outlier,
format_value(unit, outlier_max)))
print(" sample1(%s): Q1=%s mean=%s Q3=%s stdev=%s MAD=%s"
% (len(sample1),
sample1_str[0],
sample1_str[1],
sample1_str[2],
sample1_str[3],
sample1_str[4]))
print(" sample2(%s): Q1=%s mean=%s Q3=%s stdev=%s MAD=%s"
% (len(sample2),
sample2_str[0],
sample2_str[1],
sample2_str[2],
sample2_str[3],
sample2_str[4]))
print(" diff: Q1=%+.0f%% mean=%+.0f%% Q3=%+.0f%% stdev=%+.0f%% MAD=%+.0f%%"
% (q1_diff * 100,
mean_diff * 100,
q3_diff * 100,
stdev_diff * 100,
mad_diff * 100))
if outlier:
return False
if not(-0.5 <= mean_diff <= 0.10):
return False
if abs(mad_diff) > 0.10:
return False
if abs(q1_diff) > 0.05:
return False
if abs(q3_diff) > 0.05:
return False
return True
def format_run(bench,
run_index,
run,
common_metadata=None,
raw=False,
verbose=0,
lines=None):
if lines is None:
lines = []
inner_loops = run.get_inner_loops()
if run._is_calibration():
if run._is_calibration_warmups():
warmups = run._get_calibration_warmups()
action = 'calibrate the number of warmups: %s' % format_number(
warmups)
elif run._is_recalibration_warmups():
warmups = run._get_calibration_warmups()
action = 'recalibrate the number of warmups: %s' % format_number(
warmups)
elif run._is_recalibration_loops():
loops = run._get_calibration_loops()
action = 'recalibrate the number of loops: %s' % format_number(
loops)
else:
loops = run._get_calibration_loops()
action = 'calibrate the number of loops: %s' % format_number(loops)
lines.append("Run %s: %s" % (run_index, action))
if raw:
name = 'raw calibrate'
else:
name = 'calibrate'
unit = bench.get_unit()
format_value = bench.format_value
for index, warmup in enumerate(run.warmups, 1):
loops, value = warmup
raw_value = value * (loops * inner_loops)
if raw:
text = format_value(raw_value)
text = ("%s (loops: %s)" %
(format_value(raw_value), format_number(loops)))
# when using --track-memory, displaying value * loops doesn't make
# sense, so only display raw value if the unit is seconds
elif unit == 'second':
text = ("%s (loops: %s, raw: %s)" %
(format_value(value), format_number(loops),
format_value(raw_value)))
else:
text = ("%s (loops: %s)" %
(format_value(value), format_number(loops)))
lines.append("- %s %s: %s" % (name, index, text))
else:
show_warmup = (verbose >= 0)
total_loops = run.get_total_loops()
values = run.values
if raw:
warmups = [
bench.format_value(value * (loops * inner_loops))
for loops, value in run.warmups
]
values = [value * total_loops for value in values]
else:
warmups = run.warmups
if warmups:
warmups = [value for loops, value in warmups]
warmups = _format_values_diff(bench, warmups, raw, total_loops)
values = _format_values_diff(bench, values, raw, total_loops)
if verbose >= 0:
loops = run.get_loops()
lines.append("Run %s: %s, %s, %s" %
(run_index, format_number(len(warmups), 'warmup'),
format_number(len(values), 'value'),
format_number(loops, 'loop')))
else:
lines.append("Run %s:" % run_index)
if warmups and show_warmup:
if raw:
name = 'raw warmup'
else:
name = 'warmup'
for index, warmup in enumerate(warmups, 1):
lines.append('- %s %s: %s' % (name, index, warmup))
if raw:
name = 'raw value'
else:
name = 'value'
for index, value in enumerate(values, 1):
lines.append('- %s %s: %s' % (name, index, value))
if verbose > 0:
metadata = run.get_metadata()
if metadata:
lines.append('- Metadata:')
for name, value in sorted(metadata.items()):
if common_metadata and name in common_metadata:
continue
value = _format_metadata(name, value)
lines.append(' %s: %s' % (name, value))
return lines
def format_stats(bench, lines):
fmt = bench.format_value
values = bench.get_values()
nrun = bench.get_nrun()
nvalue = len(values)
empty_line(lines)
# Total duration
duration = bench.get_total_duration()
if duration:
lines.append("Total duration: %s" % format_seconds(duration))
# Start/End dates
dates = bench.get_dates()
if dates:
start, end = dates
lines.append("Start date: %s" %
format_datetime(start, microsecond=False))
lines.append("End date: %s" % format_datetime(end, microsecond=False))
# Raw value minimize/maximum
raw_values = bench._get_raw_values()
lines.append("Raw value minimum: %s" % bench.format_value(min(raw_values)))
lines.append("Raw value maximum: %s" % bench.format_value(max(raw_values)))
lines.append('')
# Number of values
ncalibration_runs = sum(run._is_calibration() for run in bench._runs)
lines.append("Number of calibration run: %s" %
format_number(ncalibration_runs))
lines.append("Number of run with values: %s" %
(format_number(nrun - ncalibration_runs)))
lines.append("Total number of run: %s" % format_number(nrun))
lines.append('')
# Number of values
nwarmup = bench._get_nwarmup()
text = format_number(nwarmup)
if isinstance(nwarmup, float):
text += ' (average)'
lines.append('Number of warmup per run: %s' % text)
nvalue_per_run = bench._get_nvalue_per_run()
text = format_number(nvalue_per_run)
if isinstance(nvalue_per_run, float):
text += ' (average)'
lines.append('Number of value per run: %s' % text)
# Loop iterations per value
loops = bench.get_loops()
inner_loops = bench.get_inner_loops()
total_loops = loops * inner_loops
if isinstance(total_loops, int):
text = format_number(total_loops)
else:
text = "%s (average)" % total_loops
if not (isinstance(inner_loops, int) and inner_loops == 1):
if isinstance(loops, int):
loops = format_number(loops, 'outer-loop')
else:
loops = '%.1f outer-loops (average)'
if isinstance(inner_loops, int):
inner_loops = format_number(inner_loops, 'inner-loop')
else:
inner_loops = "%.1f inner-loops (average)" % inner_loops
text = '%s (%s x %s)' % (text, loops, inner_loops)
lines.append("Loop iterations per value: %s" % text)
lines.append("Total number of values: %s" % format_number(nvalue))
lines.append('')
# Minimum
table = []
table.append(("Minimum", bench.format_value(min(values))))
# Median +- MAD
median = bench.median()
if len(values) > 2:
median_abs_dev = bench.median_abs_dev()
table.append(("Median +- MAD", "%s +- %s" % bench.format_values(
(median, median_abs_dev))))
else:
table.append(("Mean", bench.format_value(median)))
# Mean +- std dev
mean = bench.mean()
if len(values) > 2:
stdev = bench.stdev()
table.append(("Mean +- std dev", "%s +- %s" % bench.format_values(
(mean, stdev))))
else:
table.append(("Mean", bench.format_value(mean)))
table.append(("Maximum", bench.format_value(max(values))))
# Render table
width = max(len(row[0]) + 1 for row in table)
for key, value in table:
key = (key + ':').ljust(width)
lines.append("%s %s" % (key, value))
lines.append('')
def format_limit(mean, value):
return ("%s (%+.0f%% of the mean)" % (fmt(value),
(value - mean) * 100.0 / mean))
# Percentiles
for p in (0, 5, 25, 50, 75, 95, 100):
text = format_limit(mean, bench.percentile(p))
text = "%3sth percentile: %s" % (p, text)
name = PERCENTILE_NAMES.get(p)
if name:
text = '%s -- %s' % (text, name)
lines.append(text)
lines.append('')
# Outliers
q1 = bench.percentile(25)
q3 = bench.percentile(75)
iqr = q3 - q1
outlier_min = (q1 - 1.5 * iqr)
outlier_max = (q3 + 1.5 * iqr)
noutlier = sum(not (outlier_min <= value <= outlier_max)
for value in values)
bounds = bench.format_values((outlier_min, outlier_max))
lines.append('Number of outlier (out of %s..%s): %s' %
(bounds[0], bounds[1], format_number(noutlier)))
return lines
