def _compute_values(self, values, nvalue,
is_warmup=False,
calibrate_loops=False,
start=0):
unit = self.metadata.get('unit')
args = self.args
if nvalue < 1:
raise ValueError("nvalue must be >= 1")
if self.loops <= 0:
raise ValueError("loops must be >= 1")
if is_warmup:
value_name = 'Warmup'
else:
value_name = 'Value'
index = 1
inner_loops = self.inner_loops
if not inner_loops:
inner_loops = 1
while True:
if index > nvalue:
break
raw_value = self.task_func(self, self.loops)
raw_value = float(raw_value)
value = raw_value / (self.loops * inner_loops)
if not value and not calibrate_loops:
raise ValueError("benchmark function returned zero")
if is_warmup:
values.append((self.loops, value))
else:
values.append(value)
if args.verbose:
text = format_value(unit, value)
if is_warmup:
text = ('%s (loops: %s, raw: %s)'
% (text,
format_number(self.loops),
format_value(unit, raw_value)))
print("%s %s: %s" % (value_name, start + index, text))
if calibrate_loops and raw_value < args.min_time:
if self.loops * 2 > MAX_LOOPS:
print("ERROR: failed to calibrate the number of loops")
print("Raw timing %s with %s is still smaller than "
"the minimum time of %s"
% (format_value(unit, raw_value),
format_number(self.loops, 'loop'),
format_timedelta(args.min_time)))
sys.exit(1)
self.loops *= 2
# need more values for the calibration
nvalue += 1
index += 1
def cmd_slowest(args):
data = load_benchmarks(args, name=False)
nslowest = args.n
use_title = (data.get_nsuite() > 1)
for item in data.iter_suites():
if use_title:
display_title(item.filename, 1)
benchs = []
for bench in item.suite:
duration = bench.get_total_duration()
benchs.append((duration, bench))
benchs.sort(key=lambda item: item[0], reverse=True)
for index, item in enumerate(benchs[:nslowest], 1):
duration, bench = item
print("#%s: %s (%s)"
% (index, bench.get_name(), format_timedelta(duration)))
def format_run(bench,
run_index,
run,
common_metadata=None,
raw=False,
verbose=0,
lines=None):
if lines is None:
lines = []
inner_loops = run.get_inner_loops()
if run._is_calibration():
if run._is_calibration_warmups():
warmups = run._get_calibration_warmups()
action = 'calibrate the number of warmups: %s' % format_number(
warmups)
elif run._is_recalibration_warmups():
warmups = run._get_calibration_warmups()
action = 'recalibrate the number of warmups: %s' % format_number(
warmups)
elif run._is_recalibration_loops():
loops = run._get_calibration_loops()
action = 'recalibrate the number of loops: %s' % format_number(
loops)
else:
loops = run._get_calibration_loops()
action = 'calibrate the number of loops: %s' % format_number(loops)
lines.append("Run %s: %s" % (run_index, action))
if raw:
name = 'raw calibrate'
else:
name = 'calibrate'
for index, warmup in enumerate(run.warmups, 1):
loops, value = warmup
raw_value = value * (loops * inner_loops)
if raw:
text = format_timedelta(raw_value)
text = ("%s (loops: %s)" %
(format_timedelta(raw_value), format_number(loops)))
else:
text = ("%s (loops: %s, raw: %s)" %
(format_timedelta(value), format_number(loops),
format_timedelta(raw_value)))
lines.append("- %s %s: %s" % (name, index, text))
else:
show_warmup = (verbose >= 0)
total_loops = run.get_total_loops()
values = run.values
if raw:
warmups = [
bench.format_value(value * (loops * inner_loops))
for loops, value in run.warmups
]
values = [value * total_loops for value in values]
else:
warmups = run.warmups
if warmups:
warmups = [value for loops, value in warmups]
warmups = _format_values_diff(bench, warmups, raw, total_loops)
values = _format_values_diff(bench, values, raw, total_loops)
if verbose >= 0:
loops = run.get_loops()
lines.append("Run %s: %s, %s, %s" %
(run_index, format_number(len(warmups), 'warmup'),
format_number(len(values), 'value'),
format_number(loops, 'loop')))
else:
lines.append("Run %s:" % run_index)
if warmups and show_warmup:
if raw:
name = 'raw warmup'
else:
name = 'warmup'
for index, warmup in enumerate(warmups, 1):
lines.append('- %s %s: %s' % (name, index, warmup))
if raw:
name = 'raw value'
else:
name = 'value'
for index, value in enumerate(values, 1):
lines.append('- %s %s: %s' % (name, index, value))
if verbose > 0:
metadata = run.get_metadata()
if metadata:
lines.append('- Metadata:')
for name, value in sorted(metadata.items()):
if common_metadata and name in common_metadata:
continue
value = _format_metadata(name, value)
lines.append(' %s: %s' % (name, value))
return lines
def collect_python_metadata(metadata):
# Implementation
impl = perf.python_implementation()
metadata['python_implementation'] = impl
# Version
version = platform.python_version()
match = re.search(r'\[(PyPy [^ ]+)', sys.version)
if match:
version = '%s (Python %s)' % (match.group(1), version)
bits = platform.architecture()[0]
if bits:
if bits == '64bit':
bits = '64-bit'
elif bits == '32bit':
bits = '32-bit'
version = '%s (%s)' % (version, bits)
# '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,'
match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version)
if match:
revision = match.group(1)
else:
# 'bbd45126bc691f669c4ebdfbd74456cd274c6b92'
# in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,'
match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version)
if match:
revision = match.group(1)
else:
revision = None
if revision:
version = '%s revision %s' % (version, revision)
metadata['python_version'] = version
if sys.executable:
metadata['python_executable'] = sys.executable
# Before PEP 393 (Python 3.3)
if sys.version_info < (3, 3):
if sys.maxunicode == 0xffff:
unicode_impl = 'UTF-16'
else:
unicode_impl = 'UCS-4'
metadata['python_unicode'] = unicode_impl
# timer
if (hasattr(time, 'perf_counter')
and perf.perf_counter == time.perf_counter):
info = time.get_clock_info('perf_counter')
metadata['timer'] = ('%s, resolution: %s'
% (info.implementation,
format_timedelta(info.resolution)))
elif perf.perf_counter == time.clock:
metadata['timer'] = 'time.clock()'
elif perf.perf_counter == time.time:
metadata['timer'] = 'time.time()'
# PYTHONHASHSEED
if os.environ.get('PYTHONHASHSEED'):
hash_seed = os.environ['PYTHONHASHSEED']
try:
if hash_seed != "random":
hash_seed = int(hash_seed)
except ValueError:
pass
else:
metadata['python_hash_seed'] = hash_seed
# CFLAGS
try:
import sysconfig
except ImportError:
pass
else:
cflags = sysconfig.get_config_var('CFLAGS')
if cflags:
cflags = normalize_text(cflags)
metadata['python_cflags'] = cflags
# GC disabled?
try:
import gc
except ImportError:
pass
else:
if not gc.isenabled():
metadata['python_gc'] = 'disabled'
def __init__(self, values=None, warmups=None, processes=None,
loops=0, min_time=0.1, metadata=None,
show_name=True,
program_args=None, add_cmdline_args=None,
_argparser=None):
# Watchdog: ensure that only once instance of Runner (or a Runner
# subclass) is created per process to prevent bad suprises
cls = self.__class__
key = id(cls)
if key in cls._created:
raise RuntimeError("only one %s instance must be created "
"per process: use the same instance to run "
"all benchmarks" % cls.__name__)
cls._created.add(key)
# Use lazy import to limit imports on 'import perf'
import argparse
has_jit = perf.python_has_jit()
if not values:
if has_jit:
# Since PyPy JIT has less processes:
# run more values per process
values = 10
else:
values = 3
if not processes:
if has_jit:
# Use less processes than non-JIT, because JIT requires more
# warmups and so each worker is slower
processes = 6
else:
processes = 20
if metadata is not None:
self.metadata = metadata
else:
self.metadata = {}
# Worker task identifier: count how many times _worker() was called,
# see the --worker-task command line option
self._worker_task = 0
# Set used to check that benchmark names are unique
self._bench_names = set()
# result of argparser.parse_args()
self.args = None
# callback used to prepare command line arguments to spawn a worker
# child process. The callback is called with prepare(runner.args, cmd).
# args must be modified in-place.
self._add_cmdline_args = add_cmdline_args
# Command list arguments to call the program: (sys.argv[0],) by
# default.
#
# For example, "python3 -m perf timeit" sets program_args to
# ('-m', 'perf', 'timeit').
if program_args:
self._program_args = program_args
else:
self._program_args = (sys.argv[0],)
self._show_name = show_name
if _argparser is not None:
parser = _argparser
else:
parser = argparse.ArgumentParser()
parser.description = 'Benchmark'
parser.add_argument('--rigorous', action="store_true",
help='Spend longer running tests '
'to get more accurate results')
parser.add_argument('--fast', action="store_true",
help='Get rough answers quickly')
parser.add_argument("--debug-single-value", action="store_true",
help="Debug mode, only compute a single value")
parser.add_argument('-p', '--processes',
type=strictly_positive, default=processes,
help='number of processes used to run benchmarks '
'(default: %s)' % processes)
parser.add_argument('-n', '--values', dest="values",
type=strictly_positive, default=values,
help='number of values per process (default: %s)'
% values)
parser.add_argument('-w', '--warmups',
type=positive_or_nul,
help='number of skipped values per run used '
'to warmup the benchmark')
parser.add_argument('-l', '--loops',
type=positive_or_nul, default=loops,
help='number of loops per value, 0 means '
'automatic calibration (default: %s)'
% loops)
parser.add_argument('-v', '--verbose', action="store_true",
help='enable verbose mode')
parser.add_argument('-q', '--quiet', action="store_true",
help='enable quiet mode')
parser.add_argument('--pipe', type=int, metavar="FD",
help='Write benchmarks encoded as JSON '
'into the pipe FD')
parser.add_argument('-o', '--output', metavar='FILENAME',
help='write results encoded to JSON into FILENAME')
parser.add_argument('--append', metavar='FILENAME',
help='append results encoded to JSON into FILENAME')
parser.add_argument('--min-time', type=float, default=min_time,
help='Minimum duration in seconds of a single '
'value, used to calibrate the number of '
'loops (default: %s)'
% format_timedelta(min_time))
parser.add_argument('--worker', action='store_true',
help='Worker process, run the benchmark.')
parser.add_argument('--worker-task', type=positive_or_nul, metavar='TASK_ID',
help='Identifier of the worker task: '
'only execute the benchmark function TASK_ID')
parser.add_argument('--calibrate-loops', action="store_true",
help="calibrate the number of loops")
parser.add_argument('--recalibrate-loops', action="store_true",
help="recalibrate the the number of loops")
parser.add_argument('--calibrate-warmups', action="store_true",
help="calibrate the number of warmups")
parser.add_argument('--recalibrate-warmups', action="store_true",
help="recalibrate the number of warmups")
parser.add_argument('-d', '--dump', action="store_true",
help='display benchmark run results')
parser.add_argument('--metadata', '-m', action="store_true",
help='show metadata')
parser.add_argument('--hist', '-g', action="store_true",
help='display an histogram of values')
parser.add_argument('--stats', '-t', action="store_true",
help='display statistics (min, max, ...)')
parser.add_argument("--affinity", metavar="CPU_LIST", default=None,
help='Specify CPU affinity for worker processes. '
'This way, benchmarks can be forced to run '
'on a given set of CPUs to minimize run to '
'run variation. By default, worker processes '
'are pinned to isolate CPUs if isolated CPUs '
'are found.')
parser.add_argument("--inherit-environ", metavar='VARS',
type=comma_separated,
help='Comma-separated list of environment '
'variables inherited by worker child '
'processes.')
parser.add_argument("--no-locale",
dest="locale", action="store_false", default=True,
help="Don't copy locale environment variables "
"like LANG or LC_CTYPE.")
parser.add_argument("--python", default=sys.executable,
help='Python executable '
'(default: use running Python, '
'sys.executable)')
parser.add_argument("--compare-to", metavar="REF_PYTHON",
help='Run benchmark on the Python executable REF_PYTHON, '
'run benchmark on Python executable PYTHON, '
'and then compare REF_PYTHON result to PYTHON result')
parser.add_argument("--python-names", metavar="REF_NAME:CHANGED_NAMED",
type=parse_python_names,
help='option used with --compare-to to name '
'PYTHON as CHANGED_NAME '
'and REF_PYTHON as REF_NAME in results')
memory = parser.add_mutually_exclusive_group()
memory.add_argument('--tracemalloc', action="store_true",
help='Trace memory allocations using tracemalloc')
memory.add_argument('--track-memory', action="store_true",
help='Track memory usage using a thread')
self.argparser = parser
def collect_python_metadata(metadata):
# Implementation
impl = perf.python_implementation()
metadata['python_implementation'] = impl
# Version
version = platform.python_version()
match = re.search(r'\[(PyPy [^ ]+)', sys.version)
if match:
version = '%s (Python %s)' % (match.group(1), version)
bits = platform.architecture()[0]
if bits:
if bits == '64bit':
bits = '64-bit'
elif bits == '32bit':
bits = '32-bit'
version = '%s (%s)' % (version, bits)
# '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,'
match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version)
if match:
revision = match.group(1)
else:
# 'bbd45126bc691f669c4ebdfbd74456cd274c6b92'
# in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,'
match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version)
if match:
revision = match.group(1)
else:
revision = None
if revision:
version = '%s revision %s' % (version, revision)
metadata['python_version'] = version
if sys.executable:
metadata['python_executable'] = sys.executable
# Before PEP 393 (Python 3.3)
if sys.version_info < (3, 3):
if sys.maxunicode == 0xffff:
unicode_impl = 'UTF-16'
else:
unicode_impl = 'UCS-4'
metadata['python_unicode'] = unicode_impl
# timer
if (hasattr(time, 'perf_counter')
and perf.perf_counter == time.perf_counter):
info = time.get_clock_info('perf_counter')
metadata['timer'] = ('%s, resolution: %s'
% (info.implementation,
format_timedelta(info.resolution)))
elif perf.perf_counter == time.clock:
metadata['timer'] = 'time.clock()'
elif perf.perf_counter == time.time:
metadata['timer'] = 'time.time()'
# PYTHONHASHSEED
if os.environ.get('PYTHONHASHSEED'):
hash_seed = os.environ['PYTHONHASHSEED']
try:
if hash_seed != "random":
hash_seed = int(hash_seed)
except ValueError:
pass
else:
metadata['python_hash_seed'] = hash_seed
# CFLAGS
try:
import sysconfig
except ImportError:
pass
else:
cflags = sysconfig.get_config_var('CFLAGS')
if cflags:
cflags = normalize_text(cflags)
metadata['python_cflags'] = cflags
# GC disabled?
try:
import gc
except ImportError:
pass
else:
if not gc.isenabled():
metadata['python_gc'] = 'disabled'
def format_run(bench,
run_index,
run,
common_metadata=None,
raw=False,
verbose=0,
lines=None):
if lines is None:
lines = []
inner_loops = run._get_inner_loops()
if run._is_calibration():
lines.append("Run %s: calibrate" % (run_index, ))
for loops, value in run.warmups:
raw_value = value * (loops * inner_loops)
if raw:
lines.append("- %s: %s" % (format_number(
loops, 'loop'), format_timedelta(raw_value)))
else:
lines.append(
"- %s: %s (raw: %s)" %
(format_number(loops, 'loop'), format_timedelta(value),
format_timedelta(raw_value)))
return lines
show_warmup = (verbose >= 0)
total_loops = run.get_total_loops()
def format_values(values):
values_str = [bench.format_value(value) for value in values]
mean = bench.mean()
max_delta = mean * 0.05
for index, value in enumerate(values):
if raw:
value = float(value) / total_loops
delta = float(value) - mean
if abs(delta) > max_delta:
values_str[index] += ' (%+.0f%%)' % (delta * 100 / mean)
return values_str
values = run.values
if raw:
warmups = [
bench.format_value(value * (loops * inner_loops))
for loops, value in run.warmups
]
values = [value * total_loops for value in values]
else:
warmups = run.warmups
if warmups:
warmups = [value for loops, value in warmups]
warmups = format_values(warmups)
values = format_values(values)
if raw:
name = 'raw values'
else:
name = 'values'
text = '%s (%s): %s' % (name, len(values), ', '.join(values))
if warmups and show_warmup:
if raw:
name = 'raw warmups'
else:
name = 'warmups'
text = ('%s (%s): %s; %s' %
(name, len(warmups), ', '.join(warmups), text))
text = "Run %s: %s" % (run_index, text)
lines.append(text)
if verbose > 0:
prefix = ' '
metadata = run.get_metadata()
for name, value in sorted(metadata.items()):
if common_metadata and name in common_metadata:
continue
value = _format_metadata(name, value)
lines.append('%s%s: %s' % (prefix, name, value))
return lines