def test_compare_to(self): runs = self.create_runs((1.0, 1.5, 2.0), { 'hostname': 'toto', 'python_version': '2.7' }) ref_result = perf.Benchmark(runs=runs, name='py2') runs = self.create_runs((1.5, 2.0, 2.5), { 'hostname': 'toto', 'python_version': '3.4' }) changed_result = perf.Benchmark(runs=runs, name='py3') stdout = self.compare('compare_to', ref_result, changed_result) expected = ('Reference: py2\n' 'Changed: py3\n' '\n' 'Common metadata:\n' '- hostname: toto\n' '\n' 'py2 metadata:\n' '- python_version: 2.7\n' '\n' 'py3 metadata:\n' '- python_version: 3.4\n' '\n' 'Average: [py2] 1.50 sec +- 0.50 sec ' '-> [py3] 2.00 sec +- 0.50 sec: 1.3x slower\n' 'Not significant!') self.assertEqual(stdout.rstrip(), expected)
def test_add_runs(self): values1 = (1.0, 2.0, 3.0) bench = perf.Benchmark([create_run(values1)]) values2 = (4.0, 5.0, 6.0) bench2 = perf.Benchmark([create_run(values2)]) bench.add_runs(bench2) self.assertEqual(bench.get_values(), values1 + values2)
def test_get_total_duration(self): run = create_run([1.0]) bench = perf.Benchmark([run]) suite = perf.BenchmarkSuite([bench]) run = create_run([2.0]) bench = perf.Benchmark([run]) suite.add_runs(bench) self.assertEqual(suite.get_total_duration(), 3.0)
def test_get_dates(self): bench = perf.Benchmark([create_run()]) self.assertIsNone(bench.get_dates()) metadata = {'date': '2016-07-20T14:06:00', 'duration': 60.0} bench = perf.Benchmark([create_run(metadata=metadata)]) self.assertEqual(bench.get_dates(), (datetime.datetime( 2016, 7, 20, 14, 6, 0), datetime.datetime(2016, 7, 20, 14, 7, 0))) metadata = {'date': '2016-07-20T14:10:00', 'duration': 60.0} bench.add_run(create_run(metadata=metadata)) self.assertEqual(bench.get_dates(), (datetime.datetime( 2016, 7, 20, 14, 6, 0), datetime.datetime(2016, 7, 20, 14, 11, 0)))
def test__get_nvalue_per_run(self): # exact runs = [create_run([1.0, 2.0, 3.0]), create_run([4.0, 5.0, 6.0])] bench = perf.Benchmark(runs) nvalue = bench._get_nvalue_per_run() self.assertEqual(nvalue, 3) self.assertIsInstance(nvalue, int) # average runs = [create_run([1.0, 2.0, 3.0, 4.0]), create_run([5.0, 6.0])] bench = perf.Benchmark(runs) nvalue = bench._get_nvalue_per_run() self.assertEqual(nvalue, 3.0) self.assertIsInstance(nvalue, float)
def test_add_runs(self): # bench 1 values = (1.0, 2.0, 3.0) run = perf.Run(values, metadata={'name': "bench"}) bench = perf.Benchmark([run]) suite = perf.BenchmarkSuite([bench]) # bench 2 values2 = (4.0, 5.0, 6.0) run = perf.Run(values2, metadata={'name': "bench"}) bench2 = perf.Benchmark([run]) suite.add_runs(bench2) bench = suite.get_benchmark('bench') self.assertEqual(bench.get_values(), values + values2)
def test_compare(self): runs = self.create_runs((1.0, 1.5, 2.0)) ref_result = perf.Benchmark(runs=runs, name='py2') runs = self.create_runs((1.5, 2.0, 2.5)) changed_result = perf.Benchmark(runs=runs, name='py3') stdout = self.compare('compare', ref_result, changed_result) expected = ('Reference (best): py2\n' '\n' 'Average: [py2] 1.50 sec +- 0.50 sec ' '-> [py3] 2.00 sec +- 0.50 sec: 1.3x slower\n' 'Not significant!') self.assertEqual(stdout.rstrip(), expected)
def test_stats_empty(self): run = create_run(values=[], warmups=[(4, 10.0)]) bench = perf.Benchmark([run]) self.assertRaises(Exception, bench.mean) self.assertRaises(Exception, bench.median) self.assertRaises(Exception, bench.stdev) self.assertRaises(Exception, bench.median_abs_dev)
def test_get_warmups(self): # exact runs = [create_run((1.0, 2.0, 3.0), warmups=[(1, 1.0)]), create_run((5.0, 6.0), warmups=[(1, 4.0)])] bench = perf.Benchmark(runs) nwarmup = bench._get_nwarmup() self.assertEqual(nwarmup, 1) self.assertIsInstance(nwarmup, int) # average runs = [create_run([3.0], warmups=[(1, 1.0), (1, 2.0)]), create_run([4.0, 5.0, 6.0])] bench = perf.Benchmark(runs) nwarmup = bench._get_nwarmup() self.assertEqual(nwarmup, 1) self.assertIsInstance(nwarmup, float)
def _worker(self, sample_func): loops = self.args.loops if loops < 1: # FIXME: move this check in argument parsing raise ValueError("--loops must be >= 1") run_result = perf.RunResult(loops=loops, inner_loops=self.inner_loops, metadata=self.metadata) # only import metadata submodule in worker processes from perf import metadata as perf_metadata perf_metadata.collect_metadata(run_result.metadata) for is_warmup, run in self._range(): dt = sample_func(loops) dt = float(dt) / loops if self.inner_loops is not None: dt /= self.inner_loops self._add(run_result, is_warmup, run, dt) self._display_run_result_avg(run_result) result = perf.Benchmark(name=self.name) result.runs.append(run_result) return result
def cmd_collect_metadata(args): filename = args.output if filename and os.path.exists(filename): print("ERROR: The JSON file %r already exists" % filename) sys.exit(1) cpus = args.affinity if cpus: if not set_cpu_affinity(cpus): print("ERROR: failed to set the CPU affinity") sys.exit(1) else: cpus = get_isolated_cpus() if cpus: set_cpu_affinity(cpus) # ignore if set_cpu_affinity() failed run = perf.Run([1.0]) metadata = run.get_metadata() if metadata: print("Metadata:") for line in format_metadata(metadata): print(line) if filename: run = run._update_metadata({'name': 'metadata'}) bench = perf.Benchmark([run]) bench.dump(filename)
def _worker(self, task): self._cpu_affinity() self._process_priority() run = task.create_run() bench = perf.Benchmark((run,)) self._display_result(bench, checks=False) return bench
def _spawn_workers(self): verbose = self.args.verbose stream = self._stream() nprocess = self.args.processes bench = perf.Benchmark(name=self.name) for process in range(nprocess): run = self._spawn_worker() bench.runs.append(run) if verbose > 1: text = perf._very_verbose_run(run) print("Run %s/%s: %s" % (1 + process, nprocess, text), file=stream) else: print(".", end='', file=stream) stream.flush() if verbose <= 1: print(file=stream) if self.args.metadata: perf._display_metadata(bench.get_metadata(), file=stream) perf._display_benchmark_avg(bench, verbose=verbose, file=stream) stream.flush() _json_dump(bench, self.args) return bench
def test_stats(self): values = [float(value) for value in range(1, 96)] run = create_run(values) bench = perf.Benchmark([run]) self.assertEqual(bench.mean(), 48.0) self.assertEqual(bench.median(), 48.0) self.assertAlmostEqual(bench.stdev(), 27.5680, delta=1e-3) self.assertEqual(bench.median_abs_dev(), 24.0)
def test_stats_single(self): values = [7.0] run = create_run(values) bench = perf.Benchmark([run]) self.assertEqual(bench.mean(), 7.0) self.assertEqual(bench.median(), 7.0) self.assertRaises(Exception, bench.stdev) self.assertEqual(bench.median_abs_dev(), 0.0)
def test_stats_same(self): values = [5.0 for i in range(10)] run = create_run(values) bench = perf.Benchmark([run]) self.assertEqual(bench.mean(), 5.0) self.assertEqual(bench.median(), 5.0) self.assertEqual(bench.stdev(), 0.0) self.assertEqual(bench.median_abs_dev(), 0.0)
def test_format_result_calibration(self): run = perf.Run([], warmups=[(100, 1.0)], metadata={'name': 'bench', 'loops': 100}, collect_metadata=False) bench = perf.Benchmark([run]) self.assertEqual(cli.format_result_value(bench), '<calibration: 100 loops>') self.assertEqual(cli.format_result(bench), 'Calibration: 100 loops') self.assertRaises(ValueError, bench.median)
def test_get_unit(self): run = perf.Run((1.0, ), metadata={ 'name': 'bench', 'unit': 'byte' }, collect_metadata=False) bench = perf.Benchmark([run]) self.assertEqual(bench.get_unit(), 'byte')
def test_compare_to(self): def time_func(loops): return 1.0 def abs_executable(python): return python run = perf.Run([1.5], metadata={'name': 'name'}, collect_metadata=False) bench = perf.Benchmark([run]) suite = perf.BenchmarkSuite([bench]) with ExitStack() as cm: def popen(*args, **kw): mock_popen = mock.Mock() mock_popen.wait.return_value = 0 return mock_popen mock_subprocess = cm.enter_context( mock.patch('perf._runner.subprocess')) mock_subprocess.Popen.side_effect = popen cm.enter_context( mock.patch('perf._runner.abs_executable', side_effect=abs_executable)) cm.enter_context( mock.patch('perf._runner._load_suite_from_pipe', return_value=suite)) runner = perf.Runner() args = [ "--python=python1", "--compare-to=python2", "--min-time=5", "-p1", "-w3", "-n7", "-l11" ] runner.parse_args(args) with tests.capture_stdout(): runner.bench_time_func('name', time_func) def popen_call(python): args = [ python, mock.ANY, '--worker', '--pipe', mock.ANY, '--worker-task=0', '--values', '7', '--warmups', '3', '--loops', '11', '--min-time', '5.0' ] kw = {} if MS_WINDOWS: kw['close_fds'] = False elif six.PY3: kw['pass_fds'] = mock.ANY return mock.call(args, env=mock.ANY, **kw) call1 = popen_call('python2') call2 = popen_call('python1') mock_subprocess.Popen.assert_has_calls([call1, call2])
def test_get_dates(self): run = create_run(metadata={'date': '2016-07-20T14:06:00', 'duration': 60.0, 'name': 'bench1'}) bench = perf.Benchmark([run]) suite = perf.BenchmarkSuite([bench]) self.assertEqual(suite.get_dates(), (datetime.datetime(2016, 7, 20, 14, 6, 0), datetime.datetime(2016, 7, 20, 14, 7, 0))) run = create_run(metadata={'date': '2016-07-20T14:10:00', 'duration': 60.0, 'name': 'bench2'}) bench = perf.Benchmark([run]) suite.add_benchmark(bench) self.assertEqual(suite.get_dates(), (datetime.datetime(2016, 7, 20, 14, 6, 0), datetime.datetime(2016, 7, 20, 14, 11, 0)))
def create_bench(self, values, metadata=None): if metadata is None: metadata = {'name': 'bench'} elif 'name' not in metadata: metadata['name'] = 'bench' runs = [] for value in values: run = perf.Run([value], metadata=metadata, collect_metadata=False) runs.append(run) return perf.Benchmark(runs)
def test_format_result(self): run = perf.Run([1.0, 1.5, 2.0], warmups=[(1, 3.0)], metadata={'name': 'mybench'}, collect_metadata=False) bench = perf.Benchmark([run]) self.assertEqual(cli.format_result_value(bench), '1.50 sec +- 0.50 sec') self.assertEqual(cli.format_result(bench), 'Mean +- std dev: 1.50 sec +- 0.50 sec')
def test_get_total_duration(self): # use duration metadata runs = [create_run([0.1], metadata={'duration': 1.0}), create_run([0.1], metadata={'duration': 2.0})] bench = perf.Benchmark(runs) self.assertEqual(bench.get_total_duration(), 3.0) # run without duration metadata bench.add_run(create_run([5.0], metadata={})) self.assertEqual(bench.get_total_duration(), 8.0)
def test_remove_all_metadata(self): run = perf.Run((1.0,), metadata={'name': 'bench', 'os': 'win', 'unit': 'byte'}, collect_metadata=False) bench = perf.Benchmark([run]) self.assertEqual(bench.get_metadata(), {'name': 'bench', 'os': 'win', 'unit': 'byte'}) bench._remove_all_metadata() self.assertEqual(bench.get_metadata(), {'name': 'bench', 'unit': 'byte'})
def test_get_metadata(self): benchmarks = [] for name in ('a', 'b'): run = perf.Run([1.0], metadata={'name': name, 'os': 'linux'}, collect_metadata=False) bench = perf.Benchmark([run]) benchmarks.append(bench) suite = perf.BenchmarkSuite(benchmarks) self.assertEqual(suite.get_metadata(), {'os': 'linux'})
def test_update_metadata(self): runs = [] for value in (1.0, 2.0, 3.0): runs.append(perf.Run((value,), metadata={'name': 'bench'}, collect_metadata=False)) bench = perf.Benchmark(runs) self.assertEqual(bench.get_metadata(), {'name': 'bench'}) bench.update_metadata({'os': 'linux'}) self.assertEqual(bench.get_metadata(), {'os': 'linux', 'name': 'bench'})
def test_extract_metadata(self): warmups = ((1, 5.0),) runs = [perf.Run((1.0,), warmups=warmups, metadata={'name': 'bench', 'mem_usage': 5}, collect_metadata=False), perf.Run((2.0,), warmups=warmups, metadata={'name': 'bench', 'mem_usage': 13}, collect_metadata=False)] bench = perf.Benchmark(runs) bench._extract_metadata('mem_usage') self.assertEqual(bench.get_values(), (5, 13)) for run in bench.get_runs(): self.assertEqual(run.warmups, ())
def test_results(self): runs = [] for sample in (1.0, 1.5, 2.0): run = perf.RunResult([sample]) run.metadata['key'] = 'value' runs.append(run) results = perf.Benchmark(runs, "name") self.assertEqual(results.runs, runs) self.assertEqual(results.name, "name") self.assertEqual(results.get_metadata(), {'key': 'value'}) self.assertEqual(str(results), 'name: 1.50 sec +- 0.50 sec') self.assertEqual(results.format(0), '1.50 sec +- 0.50 sec') self.assertEqual(results.format(1), '1.50 sec +- 0.50 sec ' '(3 runs x 1 sample)')
def main(): testables = ( Dummy, PyListAppend, PyListPrepend, PyDeque, Logger, AppendToFile, AppendToOpenedFile, SQliteBufferedList, SQliteBufferedReverseList, SQliteBufferedDeque, SQlitePeriodicCommit, SQliteSingleTransaction, # SQlite, SQliteRamSingleTransaction, SQliteRam) print(perf.Benchmark(testables, 300000).as_text())
def test_remove_warmups(self): values = [1.0, 2.0, 3.0] raw_values = [5.0] + values run = perf.Run(values, warmups=[(1, 5.0)], metadata={'name': 'bench'}) bench = perf.Benchmark([run]) self.assertEqual(bench._get_nwarmup(), 1) self.assertEqual(bench._get_raw_values(warmups=True), raw_values) with tests.temporary_directory() as tmpdir: filename = os.path.join(tmpdir, 'test.json') bench.dump(filename) stdout = self.run_command('convert', filename, '--remove-warmups', '--stdout') bench2 = perf.Benchmark.loads(stdout) self.assertEqual(bench2._get_nwarmup(), 0) self.assertEqual(bench2._get_raw_values(warmups=True), raw_values[1:])