def run_benchmarks(bench_funcs, should_run, cmd_prefix, options):
suite = perf.BenchmarkSuite()
to_run = list(sorted(should_run))
run_count = str(len(to_run))
for index, name in enumerate(to_run):
func = bench_funcs[name]
print("[%s/%s] %s..." %
(str(index + 1).rjust(len(run_count)), run_count, name))
sys.stdout.flush()
def add_bench(dest_suite, obj):
if isinstance(obj, perf.BenchmarkSuite):
benchmarks = obj
else:
benchmarks = (obj, )
version = performance.__version__
for bench in benchmarks:
bench.update_metadata({'performance_version': version})
dest_suite.add_benchmark(bench)
try:
bench = func(cmd_prefix, options)
except Exception as exc:
print("ERROR: Benchmark %s failed: %s" % (name, exc))
sys.exit(1)
add_bench(suite, bench)
print()
return suite
def cmd_run(parser, options, bench_funcs, bench_groups):
print("Python benchmark suite %s" % performance.__version__)
print()
base = sys.executable
# Get the full path since child processes are run in an empty environment
# without the PATH variable
base = which(base)
if options.output:
check_existing(options.output)
options.base_binary = base
if not options.control_label:
options.control_label = options.base_binary
base_args = options.args.split()
base_cmd_prefix = [base] + base_args
logging.basicConfig(level=logging.INFO)
should_run = ParseBenchmarksOption(options.benchmarks, bench_groups,
options.fast or options.debug_single_sample)
should_run = FilterBenchmarks(should_run, bench_funcs, base_cmd_prefix)
base_suite = perf.BenchmarkSuite()
to_run = list(sorted(should_run))
run_count = str(len(to_run))
for index, name in enumerate(to_run):
func = bench_funcs[name]
print("[%s/%s] %s..." %
(str(index+1).rjust(len(run_count)), run_count, name))
options.benchmark_name = name # Easier than threading this everywhere.
def add_bench(dest_suite, bench):
if isinstance(bench, perf.BenchmarkSuite):
benchmarks = bench.get_benchmarks()
for bench in benchmarks:
dest_suite.add_benchmark(bench)
else:
dest_suite.add_benchmark(bench)
bench = func(base_cmd_prefix, options)
add_bench(base_suite, bench)
print()
print("Report on %s" % " ".join(platform.uname()))
if multiprocessing:
print("Total CPU cores:", multiprocessing.cpu_count())
if options.output:
base_suite.dump(options.output)
if options.append:
perf.add_runs(options.append, base_suite)
display_suite(base_suite)
def test_compare_to(self):
def time_func(loops):
return 1.0
def abs_executable(python):
return python
run = perf.Run([1.5],
metadata={'name': 'name'},
collect_metadata=False)
bench = perf.Benchmark([run])
suite = perf.BenchmarkSuite([bench])
with ExitStack() as cm:
def popen(*args, **kw):
mock_popen = mock.Mock()
mock_popen.wait.return_value = 0
return mock_popen
mock_subprocess = cm.enter_context(
mock.patch('perf._runner.subprocess'))
mock_subprocess.Popen.side_effect = popen
cm.enter_context(
mock.patch('perf._runner.abs_executable',
side_effect=abs_executable))
cm.enter_context(
mock.patch('perf._runner._load_suite_from_pipe',
return_value=suite))
runner = perf.Runner()
args = [
"--python=python1", "--compare-to=python2", "--min-time=5",
"-p1", "-w3", "-n7", "-l11"
]
runner.parse_args(args)
with tests.capture_stdout():
runner.bench_time_func('name', time_func)
def popen_call(python):
args = [
python, mock.ANY, '--worker', '--pipe', mock.ANY,
'--worker-task=0', '--values', '7', '--warmups', '3',
'--loops', '11', '--min-time', '5.0'
]
kw = {}
if MS_WINDOWS:
kw['close_fds'] = False
elif six.PY3:
kw['pass_fds'] = mock.ANY
return mock.call(args, env=mock.ANY, **kw)
call1 = popen_call('python2')
call2 = popen_call('python1')
mock_subprocess.Popen.assert_has_calls([call1, call2])
def test_get_total_duration(self):
run = create_run([1.0])
bench = perf.Benchmark([run])
suite = perf.BenchmarkSuite([bench])
run = create_run([2.0])
bench = perf.Benchmark([run])
suite.add_runs(bench)
self.assertEqual(suite.get_total_duration(), 3.0)
def test_suite(self):
telco = self.benchmark('telco')
go = self.benchmark('go')
suite = perf.BenchmarkSuite([telco, go])
self.assertIsNone(suite.filename)
self.assertEqual(len(suite), 2)
self.assertEqual(suite.get_benchmarks(), [telco, go])
self.assertEqual(suite.get_benchmark('go'), go)
with self.assertRaises(KeyError):
suite.get_benchmark('non_existent')
def test_get_metadata(self):
benchmarks = []
for name in ('a', 'b'):
run = perf.Run([1.0],
metadata={'name': name, 'os': 'linux'},
collect_metadata=False)
bench = perf.Benchmark([run])
benchmarks.append(bench)
suite = perf.BenchmarkSuite(benchmarks)
self.assertEqual(suite.get_metadata(),
{'os': 'linux'})
def create_suite(self):
bench1 = self.create_bench((1.0, 1.5, 2.0),
metadata={
'hostname': 'toto',
'python_version': '2.7',
'name': 'py2'
})
bench2 = self.create_bench((1.5, 2.0, 2.5),
metadata={
'hostname': 'toto',
'python_version': '3.4',
'name': 'py3'
})
return perf.BenchmarkSuite([bench1, bench2])
def test_add_runs(self):
# bench 1
values = (1.0, 2.0, 3.0)
run = perf.Run(values, metadata={'name': "bench"})
bench = perf.Benchmark([run])
suite = perf.BenchmarkSuite([bench])
# bench 2
values2 = (4.0, 5.0, 6.0)
run = perf.Run(values2, metadata={'name': "bench"})
bench2 = perf.Benchmark([run])
suite.add_runs(bench2)
bench = suite.get_benchmark('bench')
self.assertEqual(bench.get_values(), values + values2)
def _main(self):
start_time = perf.monotonic_clock()
self.parse_args()
self._cpu_affinity()
suite = perf.BenchmarkSuite()
try:
self._spawn_workers(suite, start_time)
except KeyboardInterrupt:
print("Interrupted: exit", file=sys.stderr)
sys.exit(1)
return suite
def __init__(self, runner, name, verbose=None):
self.name = '%04i-%02i-%02i %02i:%02i:%02i' % \
(time.localtime(time.time())[:6])
if verbose is not None:
self.verbose = verbose
# Init vars
self.tests = {}
if _debug:
print('Getting machine details...')
self.machine_details = get_machine_details()
self.suite = perf.BenchmarkSuite()
self.runner = runner
def add_bench(dest_suite, obj):
if isinstance(obj, perf.BenchmarkSuite):
benchmarks = obj
else:
benchmarks = (obj, )
version = performance.__version__
for bench in benchmarks:
bench.update_metadata({'performance_version': version})
if dest_suite is not None:
dest_suite.add_benchmark(bench)
else:
dest_suite = perf.BenchmarkSuite([bench])
return dest_suite
def test_get_dates(self):
run = create_run(metadata={'date': '2016-07-20T14:06:00',
'duration': 60.0,
'name': 'bench1'})
bench = perf.Benchmark([run])
suite = perf.BenchmarkSuite([bench])
self.assertEqual(suite.get_dates(),
(datetime.datetime(2016, 7, 20, 14, 6, 0),
datetime.datetime(2016, 7, 20, 14, 7, 0)))
run = create_run(metadata={'date': '2016-07-20T14:10:00',
'duration': 60.0,
'name': 'bench2'})
bench = perf.Benchmark([run])
suite.add_benchmark(bench)
self.assertEqual(suite.get_dates(),
(datetime.datetime(2016, 7, 20, 14, 6, 0),
datetime.datetime(2016, 7, 20, 14, 11, 0)))
def test_filter_benchmarks(self):
values = (1.0, 1.5, 2.0)
benchmarks = []
for name in ("call_simple", "go", "telco"):
bench = self.create_bench(values, metadata={'name': name})
benchmarks.append(bench)
suite = perf.BenchmarkSuite(benchmarks)
with tests.temporary_directory() as tmpdir:
filename = os.path.join(tmpdir, 'test.json')
suite.dump(filename)
stdout = self.run_command('convert', filename,
'--include-benchmark', 'go', '--stdout')
suite2 = perf.BenchmarkSuite.loads(stdout)
stdout = self.run_command('convert', filename,
'--exclude-benchmark', 'go', '--stdout')
suite3 = perf.BenchmarkSuite.loads(stdout)
self.assertEqual(suite2.get_benchmark_names(), ['go'])
self.assertEqual(suite3.get_benchmark_names(),
['call_simple', 'telco'])
def create_dummy_suite(self):
telco = self.benchmark('telco')
go = self.benchmark('go')
return perf.BenchmarkSuite([telco, go])
