def collect_metadata(process=True):
metadata = {}
metadata['perf_version'] = pyperf.__version__
metadata['date'] = format_datetime(datetime.datetime.now())
collect_system_metadata(metadata)
collect_cpu_metadata(metadata)
if process:
collect_python_metadata(metadata)
collect_memory_metadata(metadata)
return metadata
def collect_system_metadata(metadata):
metadata['platform'] = platform.platform(True, False)
if sys.platform.startswith('linux'):
collect_linux_metadata(metadata)
# on linux, load average over 1 minute
for line in read_proc("loadavg"):
fields = line.split()
loadavg = fields[0]
metadata['load_avg_1min'] = float(loadavg)
if len(fields) >= 4 and '/' in fields[3]:
runnable_threads = fields[3].split('/', 1)[0]
runnable_threads = int(runnable_threads)
metadata['runnable_threads'] = runnable_threads
if 'load_avg_1min' not in metadata and hasattr(os, 'getloadavg'):
metadata['load_avg_1min'] = os.getloadavg()[0]
# Hostname
hostname = socket.gethostname()
if hostname:
metadata['hostname'] = hostname
# Boot time
boot_time = None
for line in read_proc("stat"):
if not line.startswith("btime "):
continue
boot_time = int(line[6:])
break
if boot_time is None and psutil:
boot_time = psutil.boot_time()
if boot_time is not None:
btime = datetime.datetime.fromtimestamp(boot_time)
metadata['boot_time'] = format_datetime(btime)
metadata['uptime'] = time.time() - boot_time
def display_benchmarks(args,
show_metadata=False,
hist=False,
stats=False,
dump=False,
result=False,
checks=False,
display_runs_args=None,
only_checks=False):
data = load_benchmarks(args)
output = []
if show_metadata:
metadatas = [item.benchmark.get_metadata() for item in data]
_display_common_metadata(metadatas, lines=output)
if hist or stats or dump or show_metadata or (not result):
use_title = True
else:
use_title = False
if checks:
for index, item in enumerate(data):
warnings = format_checks(item.benchmark)
if warnings:
use_title = True
break
if use_title:
show_filename = (data.get_nsuite() > 1)
show_name = not data.has_same_unique_benchmark()
if not show_filename and stats:
show_filename = (len(data) > 1)
suite = None
for index, item in enumerate(data):
lines = []
if show_metadata:
metadata = metadatas[index]
if metadata:
empty_line(lines)
lines.append("Metadata:")
format_metadata(metadata, lines=lines)
bench_lines = format_benchmark(item.benchmark,
hist=hist,
stats=stats,
dump=dump,
checks=checks,
result=result,
display_runs_args=display_runs_args)
if bench_lines:
empty_line(lines)
lines.extend(bench_lines)
if lines:
bench_lines = lines
lines = []
if show_filename and item.suite is not suite:
suite = item.suite
format_title(item.filename, 1, lines=lines)
if stats and len(suite) > 1:
empty_line(lines)
duration = suite.get_total_duration()
lines.append("Number of benchmarks: %s" % len(suite))
lines.append("Total duration: %s" %
format_seconds(duration))
dates = suite.get_dates()
if dates:
start, end = dates
lines.append(
"Start date: %s" %
format_datetime(start, microsecond=False))
lines.append(
"End date: %s" %
format_datetime(end, microsecond=False))
if show_name:
format_title(item.name, 2, lines=lines)
empty_line(lines)
lines.extend(bench_lines)
if lines:
empty_line(output)
output.extend(lines)
for line in output:
print(line)
if not output and only_checks:
if len(data) == 1:
print("The benchmark seems to be stable")
else:
print("All benchmarks seem to be stable")
else:
for line in output:
print(line)
show_filename = (data.get_nsuite() > 1)
suite = None
for item in data:
if show_filename and item.suite is not suite:
if suite is not None:
print()
suite = item.suite
display_title(item.filename, 1)
line = format_result(item.benchmark)
if item.title:
line = '%s: %s' % (item.name, line)
print(line)
def format_stats(bench, lines):
fmt = bench.format_value
values = bench.get_values()
nrun = bench.get_nrun()
nvalue = len(values)
empty_line(lines)
# Total duration
duration = bench.get_total_duration()
if duration:
lines.append("Total duration: %s" % format_seconds(duration))
# Start/End dates
dates = bench.get_dates()
if dates:
start, end = dates
lines.append("Start date: %s" %
format_datetime(start, microsecond=False))
lines.append("End date: %s" % format_datetime(end, microsecond=False))
# Raw value minimize/maximum
raw_values = bench._get_raw_values()
lines.append("Raw value minimum: %s" % bench.format_value(min(raw_values)))
lines.append("Raw value maximum: %s" % bench.format_value(max(raw_values)))
lines.append('')
# Number of values
ncalibration_runs = sum(run._is_calibration() for run in bench._runs)
lines.append("Number of calibration run: %s" %
format_number(ncalibration_runs))
lines.append("Number of run with values: %s" %
(format_number(nrun - ncalibration_runs)))
lines.append("Total number of run: %s" % format_number(nrun))
lines.append('')
# Number of values
nwarmup = bench._get_nwarmup()
text = format_number(nwarmup)
if isinstance(nwarmup, float):
text += ' (average)'
lines.append('Number of warmup per run: %s' % text)
nvalue_per_run = bench._get_nvalue_per_run()
text = format_number(nvalue_per_run)
if isinstance(nvalue_per_run, float):
text += ' (average)'
lines.append('Number of value per run: %s' % text)
# Loop iterations per value
loops = bench.get_loops()
inner_loops = bench.get_inner_loops()
total_loops = loops * inner_loops
if isinstance(total_loops, int):
text = format_number(total_loops)
else:
text = "%s (average)" % total_loops
if not (isinstance(inner_loops, int) and inner_loops == 1):
if isinstance(loops, int):
loops = format_number(loops, 'outer-loop')
else:
loops = '%.1f outer-loops (average)'
if isinstance(inner_loops, int):
inner_loops = format_number(inner_loops, 'inner-loop')
else:
inner_loops = "%.1f inner-loops (average)" % inner_loops
text = '%s (%s x %s)' % (text, loops, inner_loops)
lines.append("Loop iterations per value: %s" % text)
lines.append("Total number of values: %s" % format_number(nvalue))
lines.append('')
# Minimum
table = []
table.append(("Minimum", bench.format_value(min(values))))
# Median +- MAD
median = bench.median()
if len(values) > 2:
median_abs_dev = bench.median_abs_dev()
table.append(("Median +- MAD", "%s +- %s" % bench.format_values(
(median, median_abs_dev))))
else:
table.append(("Mean", bench.format_value(median)))
# Mean +- std dev
mean = bench.mean()
if len(values) > 2:
stdev = bench.stdev()
table.append(("Mean +- std dev", "%s +- %s" % bench.format_values(
(mean, stdev))))
else:
table.append(("Mean", bench.format_value(mean)))
table.append(("Maximum", bench.format_value(max(values))))
# Render table
width = max(len(row[0]) + 1 for row in table)
for key, value in table:
key = (key + ':').ljust(width)
lines.append("%s %s" % (key, value))
lines.append('')
def format_limit(mean, value):
return ("%s (%+.0f%% of the mean)" % (fmt(value),
(value - mean) * 100.0 / mean))
# Percentiles
for p in (0, 5, 25, 50, 75, 95, 100):
text = format_limit(mean, bench.percentile(p))
text = "%3sth percentile: %s" % (p, text)
name = PERCENTILE_NAMES.get(p)
if name:
text = '%s -- %s' % (text, name)
lines.append(text)
lines.append('')
# Outliers
q1 = bench.percentile(25)
q3 = bench.percentile(75)
iqr = q3 - q1
outlier_min = (q1 - 1.5 * iqr)
outlier_max = (q3 + 1.5 * iqr)
noutlier = sum(not (outlier_min <= value <= outlier_max)
for value in values)
bounds = bench.format_values((outlier_min, outlier_max))
lines.append('Number of outlier (out of %s..%s): %s' %
(bounds[0], bounds[1], format_number(noutlier)))
return lines