def test_filter_benchmarks(self):
values = (1.0, 1.5, 2.0)
benchmarks = []
for name in ("call_simple", "go", "telco"):
bench = self.create_bench(values, metadata={'name': name})
benchmarks.append(bench)
suite = pyperf.BenchmarkSuite(benchmarks)
with tests.temporary_directory() as tmpdir:
filename = os.path.join(tmpdir, 'test.json')
suite.dump(filename)
stdout = self.run_command('convert', filename,
'--include-benchmark', 'go', '--stdout')
suite2 = pyperf.BenchmarkSuite.loads(stdout)
stdout = self.run_command('convert', filename,
'--exclude-benchmark', 'go', '--stdout')
suite3 = pyperf.BenchmarkSuite.loads(stdout)
self.assertEqual(suite2.get_benchmark_names(),
['go'])
self.assertEqual(suite3.get_benchmark_names(),
['call_simple', 'telco'])
def run_perf_script(level):
cmd = utils.build_command('micro_script.py')
benchmarks = []
for b in config_micro.benchmarks:
if config_micro.clear_db:
utils.drop_tables(config_micro.db_url)
server_pid = utils.start_app(level,
config_micro.webserver,
config_micro.port,
config_micro.url,
'micro.app:app',
output=config_micro.output)
time.sleep(config_micro.bm_cooldown)
benchmark_file = configparser.ConfigParser()
benchmark_file['bench'] = {'name': b[0], 'desc': b[1]}
with open('bm_info.ini', 'w') as configfile:
benchmark_file.write(configfile)
with temporary_file() as tmp:
cmd.extend(('--output', tmp))
run_command(cmd)
benchmarks.append(pyperf.Benchmark.load(tmp))
utils.stop_app(server_pid)
time.sleep(config_micro.bm_cooldown)
return pyperf.BenchmarkSuite(benchmarks)
def test_get_total_duration(self):
run = create_run([1.0])
bench = pyperf.Benchmark([run])
suite = pyperf.BenchmarkSuite([bench])
run = create_run([2.0])
bench = pyperf.Benchmark([run])
suite.add_runs(bench)
self.assertEqual(suite.get_total_duration(), 3.0)
def create_suite(self):
bench1 = self.create_bench((1.0, 1.5, 2.0),
metadata={'hostname': 'toto',
'python_version': '2.7',
'name': 'py36'})
bench2 = self.create_bench((1.5, 2.0, 2.5),
metadata={'hostname': 'toto',
'python_version': '3.4',
'name': 'py38'})
return pyperf.BenchmarkSuite([bench1, bench2])
def test_compare_to(self):
def time_func(loops):
return 1.0
def abs_executable(python):
return python
run = pyperf.Run([1.5],
metadata={'name': 'name'},
collect_metadata=False)
bench = pyperf.Benchmark([run])
suite = pyperf.BenchmarkSuite([bench])
with ExitStack() as cm:
def popen(*args, **kw):
mock_popen = mock.Mock()
mock_popen.wait.return_value = 0
return mock_popen
mock_subprocess = cm.enter_context(
mock.patch('pyperf._master.subprocess'))
mock_subprocess.Popen.side_effect = popen
cm.enter_context(
mock.patch('pyperf._runner.abs_executable',
side_effect=abs_executable))
cm.enter_context(
mock.patch('pyperf._master._load_suite_from_pipe',
return_value=suite))
args = [
"--python=python3.8", "--compare-to=python3.6", "--min-time=5",
"-p1", "-w3", "-n7", "-l11"
]
runner = self.create_runner(args)
with tests.capture_stdout():
runner.bench_time_func('name', time_func)
def popen_call(python):
args = [
python, mock.ANY, '--worker', '--pipe', mock.ANY,
'--worker-task=0', '--values', '7', '--min-time', '5.0',
'--loops', '11', '--warmups', '3'
]
kw = {}
if MS_WINDOWS:
kw['close_fds'] = False
else:
kw['pass_fds'] = mock.ANY
return mock.call(args, env=mock.ANY, **kw)
call1 = popen_call('python3.6')
call2 = popen_call('python3.8')
mock_subprocess.Popen.assert_has_calls([call1, call2])
def test_suite(self):
telco = self.benchmark('telco')
go = self.benchmark('go')
suite = pyperf.BenchmarkSuite([telco, go])
self.assertIsNone(suite.filename)
self.assertEqual(len(suite), 2)
self.assertEqual(suite.get_benchmarks(), [telco, go])
self.assertEqual(suite.get_benchmark('go'), go)
with self.assertRaises(KeyError):
suite.get_benchmark('non_existent')
def test_get_metadata(self):
benchmarks = []
for name in ('a', 'b'):
run = pyperf.Run([1.0],
metadata={'name': name, 'os': 'linux'},
collect_metadata=False)
bench = pyperf.Benchmark([run])
benchmarks.append(bench)
suite = pyperf.BenchmarkSuite(benchmarks)
self.assertEqual(suite.get_metadata(),
{'os': 'linux'})
def add_bench(dest_suite, obj):
benchmarks = obj if isinstance(obj,
pyperf.BenchmarkSuite) else (obj, )
version = pyperformance.__version__
for bench in benchmarks:
bench.update_metadata({'performance_version': version})
if dest_suite is not None:
dest_suite.add_benchmark(bench)
else:
dest_suite = pyperf.BenchmarkSuite([bench])
return dest_suite
def test_add_runs(self):
# bench 1
values = (1.0, 2.0, 3.0)
run = pyperf.Run(values, metadata={'name': "bench"})
bench = pyperf.Benchmark([run])
suite = pyperf.BenchmarkSuite([bench])
# bench 2
values2 = (4.0, 5.0, 6.0)
run = pyperf.Run(values2, metadata={'name': "bench"})
bench2 = pyperf.Benchmark([run])
suite.add_runs(bench2)
bench = suite.get_benchmark('bench')
self.assertEqual(bench.get_values(), values + values2)
def add_bench(dest_suite, obj):
if isinstance(obj, pyperf.BenchmarkSuite):
results = obj
else:
results = (obj, )
version = pyperformance.__version__
for res in results:
res.update_metadata({'performance_version': version})
if dest_suite is not None:
dest_suite.add_benchmark(res)
else:
dest_suite = pyperf.BenchmarkSuite([res])
return dest_suite
def test_get_dates(self):
run = create_run(metadata={'date': '2016-07-20T14:06:00',
'duration': 60.0,
'name': 'bench1'})
bench = pyperf.Benchmark([run])
suite = pyperf.BenchmarkSuite([bench])
self.assertEqual(suite.get_dates(),
(datetime.datetime(2016, 7, 20, 14, 6, 0),
datetime.datetime(2016, 7, 20, 14, 7, 0)))
run = create_run(metadata={'date': '2016-07-20T14:10:00',
'duration': 60.0,
'name': 'bench2'})
bench = pyperf.Benchmark([run])
suite.add_benchmark(bench)
self.assertEqual(suite.get_dates(),
(datetime.datetime(2016, 7, 20, 14, 6, 0),
datetime.datetime(2016, 7, 20, 14, 11, 0)))
def run_suite(benchmarks: List[Benchmark], pyperf_args: Sequence[str],
workdir: Path) -> Tuple[Optional[pyperf.BenchmarkSuite], bool]:
import black
results: List[pyperf.Benchmark] = []
errored = False
for i, bm in enumerate(benchmarks, start=1):
bm_type = f"{'micro' if bm.micro else ''}benchmark"
bm_count = f"({i}/{len(benchmarks)})"
log(f"Running `{bm.name}` {bm_type} {bm_count}", bold=True)
script = workdir / f"{i}.py"
script.write_text(bm.code, encoding="utf8")
result_file = workdir / f"{i}.json"
cmd = [
sys.executable,
str(script), "--output",
str(result_file), *pyperf_args
]
t0 = time.perf_counter()
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError:
err("Failed to run benchmark ^^^")
errored = True
else:
t1 = time.perf_counter()
log(f"Took {round(t1 - t0, 3)} seconds.", bold=True)
result = pyperf.Benchmark.loads(
result_file.read_text(encoding="utf8"))
# fmt: off
result.update_metadata({
"description": bm.description,
"blackbench-version": __version__,
"black-version": black.__version__
})
# fmt: on
results.append(result)
else:
if results:
return pyperf.BenchmarkSuite(results), errored
else:
return None, True
def run(self) -> pyperf.BenchmarkSuite:
results = [benchmark.run(self) for benchmark in self.benchmarks]
return pyperf.BenchmarkSuite(results)
def create_dummy_suite(self):
telco = self.benchmark('telco')
go = self.benchmark('go')
return pyperf.BenchmarkSuite([telco, go])
