def get_results():
results = util.load_json(glob.glob(join(
tmpdir, 'results_workflow', 'orangutan', '*-*.json'))[0])
# replacing NaN by 'n/a' make assertions easier
return ['n/a' if util.is_nan(item) else item
for item in results['results'][
'params_examples.track_param_selection']['result']]
def get_results():
results = util.load_json(glob.glob(join(
tmpdir, 'results_workflow', 'orangutan', '*-*.json'))[0])
# replacing NaN by 'n/a' make assertions easier
return ['n/a' if util.is_nan(item) else item
for item in results['results'][
'params_examples.track_param_selection']['result']]
def test_skip_param_selection():
d = {'repo': 'foo'}
d.update(ASV_CONF_JSON)
conf = config.Config.from_json(d)
class DummyEnv(object):
name = 'env'
d = [
{'name': 'test_nonparam', 'params': []},
{'name': 'test_param',
'params': [['1', '2', '3']],
'param_names': ['n']}
]
b = benchmarks.Benchmarks(conf, d, [r'test_nonparam', r'test_param\([23]\)'])
result = b.skip_benchmarks(DummyEnv())
assert result['test_nonparam'].result == None
assert util.is_nan(result['test_param'].result[0])
assert result['test_param'].result[1:] == [None, None]
def test_find_benchmarks(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
b = benchmarks.Benchmarks(conf, repo, envs, regex='secondary')
assert len(b) == 3
b = benchmarks.Benchmarks(conf, repo, envs, regex='example')
assert len(b) == 25
b = benchmarks.Benchmarks(conf,
repo,
envs,
regex='time_example_benchmark_1')
assert len(b) == 2
b = benchmarks.Benchmarks(conf,
repo,
envs,
regex=[
'time_example_benchmark_1',
'some regexp that does not match anything'
])
assert len(b) == 2
b = benchmarks.Benchmarks(conf, repo, envs, regex='custom')
assert sorted(b.keys()) == ['custom.time_function', 'custom.track_method']
b = benchmarks.Benchmarks(conf, repo, envs)
assert len(b) == 33
start_timestamp = datetime.datetime.utcnow()
b = benchmarks.Benchmarks(conf, repo, envs)
times = b.run_benchmarks(envs[0], profile=True, show_stderr=True)
end_timestamp = datetime.datetime.utcnow()
assert len(times) == len(b)
assert times['time_examples.TimeSuite.time_example_benchmark_1'][
'result'] != [None]
assert isinstance(
times['time_examples.TimeSuite.time_example_benchmark_1']['stats'][0]
['std'], float)
# The exact number of samples may vary if the calibration is not fully accurate
assert len(times['time_examples.TimeSuite.time_example_benchmark_1']
['samples'][0]) in (8, 9, 10)
# Benchmarks that raise exceptions should have a time of "None"
assert times['time_secondary.TimeSecondary.time_exception']['result'] == [
None
]
assert times['subdir.time_subdir.time_foo']['result'] != [None]
if not ON_PYPY:
# XXX: the memory benchmarks don't work on Pypy, since asizeof
# is CPython-only
assert times['mem_examples.mem_list']['result'][0] > 1000
assert times['time_secondary.track_value']['result'] == [42.0]
assert 'profile' in times['time_secondary.track_value']
assert 'stderr' in times['time_examples.time_with_warnings']
assert times['time_examples.time_with_warnings']['errcode'] != 0
assert times['time_examples.TimeWithBadTimer.time_it']['result'] == [0.0]
assert times['params_examples.track_param']['params'] == [[
"<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"
]]
assert times['params_examples.track_param']['result'] == [42, 42]
assert times['params_examples.mem_param']['params'] == [['10', '20'],
['2', '3']]
assert len(times['params_examples.mem_param']['result']) == 2 * 2
assert times['params_examples.ParamSuite.track_value']['params'] == [[
"'a'", "'b'", "'c'"
]]
assert times['params_examples.ParamSuite.track_value']['result'] == [
1 + 0, 2 + 0, 3 + 0
]
assert isinstance(times['params_examples.TuningTest.time_it']['result'][0],
float)
assert isinstance(times['params_examples.time_skip']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result'][1], float)
assert util.is_nan(times['params_examples.time_skip']['result'][2])
assert times['peakmem_examples.peakmem_list']['result'][0] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example']['result'] == [
500
]
assert times['cache_examples.ClassLevelSetup.track_example2'][
'result'] == [500]
assert times['cache_examples.track_cache_foo']['result'] == [42]
assert times['cache_examples.track_cache_bar']['result'] == [12]
assert times['cache_examples.track_my_cache_foo']['result'] == [0]
assert times['cache_examples.ClassLevelSetupFail.track_fail'][
'result'] == None
assert 'raise RuntimeError()' in times[
'cache_examples.ClassLevelSetupFail.track_fail']['stderr']
assert times['cache_examples.ClassLevelCacheTimeout.track_fail'][
'result'] == None
assert times['cache_examples.ClassLevelCacheTimeoutSuccess.track_success'][
'result'] == [0]
profile_path = join(tmpdir, 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value']['profile'])
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split(
) == expected
# Calibration of iterations should not rerun setup
expected = (['setup'] * 2, ['setup'] * 3)
assert times['time_examples.TimeWithRepeatCalibrate.time_it'][
'stderr'].split() in expected
# Check run time timestamps
for name, result in times.items():
assert result['started_at'] >= start_timestamp
assert result['ended_at'] >= result['started_at']
assert result['ended_at'] <= end_timestamp
def test_find_benchmarks(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='secondary')
assert len(b) == 3
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='example')
assert len(b) == 25
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='time_example_benchmark_1')
assert len(b) == 2
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex=['time_example_benchmark_1',
'some regexp that does not match anything'])
assert len(b) == 2
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash], regex='custom')
assert sorted(b.keys()) == ['custom.time_function', 'custom.track_method',
'named.track_custom_pretty_name']
assert 'pretty_name' not in b['custom.track_method']
assert b['custom.time_function']['pretty_name'] == 'My Custom Function'
assert b['named.track_custom_pretty_name']['pretty_name'] == 'this.is/the.answer'
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
assert len(b) == 35
assert 'named.OtherSuite.track_some_func' in b
start_timestamp = datetime.datetime.utcnow()
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
times = b.run_benchmarks(envs[0], profile=True, show_stderr=True)
end_timestamp = datetime.datetime.utcnow()
assert len(times) == len(b)
assert times[
'time_examples.TimeSuite.time_example_benchmark_1']['result'] != [None]
assert isinstance(times['time_examples.TimeSuite.time_example_benchmark_1']['stats'][0]['std'], float)
# The exact number of samples may vary if the calibration is not fully accurate
assert len(times['time_examples.TimeSuite.time_example_benchmark_1']['samples'][0]) >= 5
# Benchmarks that raise exceptions should have a time of "None"
assert times[
'time_secondary.TimeSecondary.time_exception']['result'] == [None]
assert times[
'subdir.time_subdir.time_foo']['result'] != [None]
if not ON_PYPY:
# XXX: the memory benchmarks don't work on Pypy, since asizeof
# is CPython-only
assert times[
'mem_examples.mem_list']['result'][0] > 1000
assert times[
'time_secondary.track_value']['result'] == [42.0]
assert 'profile' in times[
'time_secondary.track_value']
assert 'stderr' in times[
'time_examples.time_with_warnings']
assert times['time_examples.time_with_warnings']['errcode'] != 0
assert times['time_examples.TimeWithBadTimer.time_it']['result'] == [0.0]
assert times['params_examples.track_param']['params'] == [["<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"]]
assert times['params_examples.track_param']['result'] == [42, 42]
assert times['params_examples.mem_param']['params'] == [['10', '20'], ['2', '3']]
assert len(times['params_examples.mem_param']['result']) == 2*2
assert times['params_examples.ParamSuite.track_value']['params'] == [["'a'", "'b'", "'c'"]]
assert times['params_examples.ParamSuite.track_value']['result'] == [1+0, 2+0, 3+0]
assert isinstance(times['params_examples.TuningTest.time_it']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result'][1], float)
assert util.is_nan(times['params_examples.time_skip']['result'][2])
assert times['peakmem_examples.peakmem_list']['result'][0] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example']['result'] == [500]
assert times['cache_examples.ClassLevelSetup.track_example2']['result'] == [500]
assert times['cache_examples.track_cache_foo']['result'] == [42]
assert times['cache_examples.track_cache_bar']['result'] == [12]
assert times['cache_examples.track_my_cache_foo']['result'] == [0]
assert times['cache_examples.ClassLevelSetupFail.track_fail']['result'] == None
assert 'raise RuntimeError()' in times['cache_examples.ClassLevelSetupFail.track_fail']['stderr']
assert times['cache_examples.ClassLevelCacheTimeout.track_fail']['result'] == None
assert times['cache_examples.ClassLevelCacheTimeoutSuccess.track_success']['result'] == [0]
profile_path = join(tmpdir, 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value']['profile'])
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split() == expected
# Calibration of iterations should not rerun setup
expected = (['setup']*2, ['setup']*3)
assert times['time_examples.TimeWithRepeatCalibrate.time_it']['stderr'].split() in expected
# Check run time timestamps
for name, result in times.items():
assert result['started_at'] >= start_timestamp
assert result['ended_at'] >= result['started_at']
assert result['ended_at'] <= end_timestamp
def test_run_benchmarks(benchmarks_fixture, tmpdir):
conf, repo, envs, commit_hash = benchmarks_fixture
start_timestamp = datetime.datetime.utcnow()
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
# Old results to append to
results = Results.unnamed()
name = 'time_examples.TimeSuite.time_example_benchmark_1'
results.add_result(b[name],
runner.BenchmarkResult(result=[1],
samples=[[42.0, 24.0]],
number=[1],
errcode=0,
stderr='',
profile=None),
record_samples=True)
# Run
runner.run_benchmarks(b,
envs[0],
results=results,
profile=True,
show_stderr=True,
append_samples=True,
record_samples=True)
times = ResultsWrapper(results, b)
end_timestamp = datetime.datetime.utcnow()
assert len(times) == len(b)
assert times[
'time_examples.TimeSuite.time_example_benchmark_1'].result != [None]
stats = results.get_result_stats(name, b[name]['params'])
assert isinstance(stats[0]['std'], float)
# The exact number of samples may vary if the calibration is not fully accurate
samples = results.get_result_samples(name, b[name]['params'])
assert len(samples[0]) >= 4
# Explicitly provided 'prev_samples` should come first
assert samples[0][:2] == [42.0, 24.0]
# Benchmarks that raise exceptions should have a time of "None"
assert times['time_secondary.TimeSecondary.time_exception'].result == [
None
]
assert times['subdir.time_subdir.time_foo'].result != [None]
if not ON_PYPY:
# XXX: the memory benchmarks don't work on Pypy, since asizeof
# is CPython-only
assert times['mem_examples.mem_list'].result[0] > 1000
assert times['time_secondary.track_value'].result == [42.0]
assert times['time_secondary.track_value'].profile is not None
assert isinstance(times['time_examples.time_with_warnings'].stderr,
type(''))
assert times['time_examples.time_with_warnings'].errcode != 0
assert times['time_examples.TimeWithBadTimer.time_it'].result == [0.0]
assert times['params_examples.track_param'].params == [[
"<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"
]]
assert times['params_examples.track_param'].result == [42, 42]
assert times['params_examples.mem_param'].params == [['10', '20'],
['2', '3']]
assert len(times['params_examples.mem_param'].result) == 2 * 2
assert times['params_examples.ParamSuite.track_value'].params == [[
"'a'", "'b'", "'c'"
]]
assert times['params_examples.ParamSuite.track_value'].result == [
1 + 0, 2 + 0, 3 + 0
]
assert isinstance(times['params_examples.TuningTest.time_it'].result[0],
float)
assert isinstance(times['params_examples.TuningTest.time_it'].result[1],
float)
assert isinstance(times['params_examples.time_skip'].result[0], float)
assert isinstance(times['params_examples.time_skip'].result[1], float)
assert util.is_nan(times['params_examples.time_skip'].result[2])
assert times['peakmem_examples.peakmem_list'].result[0] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example'].result == [
500
]
assert times['cache_examples.ClassLevelSetup.track_example2'].result == [
500
]
assert times['cache_examples.track_cache_foo'].result == [42]
assert times['cache_examples.track_cache_bar'].result == [12]
assert times['cache_examples.track_my_cache_foo'].result == [0]
assert times['cache_examples.ClassLevelSetupFail.track_fail'].result == [
None
]
assert 'raise RuntimeError()' in times[
'cache_examples.ClassLevelSetupFail.track_fail'].stderr
assert times[
'cache_examples.ClassLevelCacheTimeout.track_fail'].result == [None]
assert times[
'cache_examples.ClassLevelCacheTimeoutSuccess.track_success'].result == [
0
]
assert times['cache_examples.time_fail_second_run'].result == [None]
assert times['cache_examples.time_fail_second_run'].samples == [None]
profile_path = join(six.text_type(tmpdir), 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value'].profile)
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it'].stderr.split(
) == expected
# Calibration of iterations should not rerun setup
expected = (['setup'] * 2, ['setup'] * 3)
assert times['time_examples.TimeWithRepeatCalibrate.time_it'].stderr.split(
) in expected
# Check tuple-form repeat attribute produced results
assert 2 <= len(times['time_examples.time_auto_repeat'].samples[0]) <= 4
# Check run time timestamps
for name, result in times.items():
assert result.started_at >= util.datetime_to_js_timestamp(
start_timestamp)
assert result.ended_at >= result.started_at
assert result.ended_at <= util.datetime_to_js_timestamp(end_timestamp)
def test_find_benchmarks(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
b = benchmarks.Benchmarks(conf, regex='secondary')
assert len(b) == 3
b = benchmarks.Benchmarks(conf, regex='example')
assert len(b) == 22
b = benchmarks.Benchmarks(conf, regex='time_example_benchmark_1')
assert len(b) == 2
b = benchmarks.Benchmarks(conf,
regex=[
'time_example_benchmark_1',
'some regexp that does not match anything'
])
assert len(b) == 2
b = benchmarks.Benchmarks(conf)
assert len(b) == 26
envs = list(environment.get_environments(conf))
b = benchmarks.Benchmarks(conf)
times = b.run_benchmarks(envs[0], profile=True, show_stderr=True)
assert len(times) == len(b)
assert times['time_examples.TimeSuite.time_example_benchmark_1'][
'result'] is not None
# Benchmarks that raise exceptions should have a time of "None"
assert times['time_secondary.TimeSecondary.time_exception'][
'result'] is None
assert times['subdir.time_subdir.time_foo']['result'] is not None
assert times['mem_examples.mem_list']['result'] > 1000
assert times['time_secondary.track_value']['result'] == 42.0
assert 'profile' in times['time_secondary.track_value']
assert 'stderr' in times['time_examples.time_with_warnings']
assert times['time_examples.time_with_warnings']['errcode'] != 0
assert times['params_examples.track_param']['result']['params'] == [[
"<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"
]]
assert times['params_examples.track_param']['result']['result'] == [42, 42]
assert times['params_examples.mem_param']['result']['params'] == [[
'10', '20'
], ['2', '3']]
assert len(times['params_examples.mem_param']['result']['result']) == 2 * 2
assert times['params_examples.ParamSuite.track_value']['result'][
'params'] == [["'a'", "'b'", "'c'"]]
assert times['params_examples.ParamSuite.track_value']['result'][
'result'] == [1 + 0, 2 + 0, 3 + 0]
assert isinstance(
times['params_examples.TuningTest.time_it']['result']['result'][0],
float)
assert isinstance(
times['params_examples.time_skip']['result']['result'][0], float)
assert isinstance(
times['params_examples.time_skip']['result']['result'][1], float)
assert util.is_nan(
times['params_examples.time_skip']['result']['result'][2])
assert times['peakmem_examples.peakmem_list']['result'] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example'][
'result'] == 500
assert times['cache_examples.ClassLevelSetup.track_example2'][
'result'] == 500
assert times['cache_examples.track_cache_foo']['result'] == 42
assert times['cache_examples.track_cache_bar']['result'] == 12
assert times['cache_examples.track_my_cache_foo']['result'] == 0
assert times['cache_examples.ClassLevelSetupFail.track_fail'][
'result'] == None
assert 'raise RuntimeError()' in times[
'cache_examples.ClassLevelSetupFail.track_fail']['stderr']
profile_path = join(tmpdir, 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value']['profile'])
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split(
) == expected
# Calibration of iterations should not rerun setup
expected = ['setup'] * 2
assert times['time_examples.TimeWithRepeatCalibrate.time_it'][
'stderr'].split() == expected
def test_find_benchmarks(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
envs = list(environment.get_environments(conf, None))
b = benchmarks.Benchmarks(conf, envs, regex='secondary')
assert len(b) == 3
b = benchmarks.Benchmarks(conf, envs, regex='example')
assert len(b) == 22
b = benchmarks.Benchmarks(conf, envs, regex='time_example_benchmark_1')
assert len(b) == 2
b = benchmarks.Benchmarks(conf, envs, regex=['time_example_benchmark_1',
'some regexp that does not match anything'])
assert len(b) == 2
b = benchmarks.Benchmarks(conf, envs)
assert len(b) == 26
b = benchmarks.Benchmarks(conf, envs)
times = b.run_benchmarks(envs[0], profile=True, show_stderr=True)
assert len(times) == len(b)
assert times[
'time_examples.TimeSuite.time_example_benchmark_1']['result'] is not None
# Benchmarks that raise exceptions should have a time of "None"
assert times[
'time_secondary.TimeSecondary.time_exception']['result'] is None
assert times[
'subdir.time_subdir.time_foo']['result'] is not None
assert times[
'mem_examples.mem_list']['result'] > 1000
assert times[
'time_secondary.track_value']['result'] == 42.0
assert 'profile' in times[
'time_secondary.track_value']
assert 'stderr' in times[
'time_examples.time_with_warnings']
assert times['time_examples.time_with_warnings']['errcode'] != 0
assert times['params_examples.track_param']['result']['params'] == [["<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"]]
assert times['params_examples.track_param']['result']['result'] == [42, 42]
assert times['params_examples.mem_param']['result']['params'] == [['10', '20'], ['2', '3']]
assert len(times['params_examples.mem_param']['result']['result']) == 2*2
assert times['params_examples.ParamSuite.track_value']['result']['params'] == [["'a'", "'b'", "'c'"]]
assert times['params_examples.ParamSuite.track_value']['result']['result'] == [1+0, 2+0, 3+0]
assert isinstance(times['params_examples.TuningTest.time_it']['result']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result']['result'][1], float)
assert util.is_nan(times['params_examples.time_skip']['result']['result'][2])
assert times['peakmem_examples.peakmem_list']['result'] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example']['result'] == 500
assert times['cache_examples.ClassLevelSetup.track_example2']['result'] == 500
assert times['cache_examples.track_cache_foo']['result'] == 42
assert times['cache_examples.track_cache_bar']['result'] == 12
assert times['cache_examples.track_my_cache_foo']['result'] == 0
assert times['cache_examples.ClassLevelSetupFail.track_fail']['result'] == None
assert 'raise RuntimeError()' in times['cache_examples.ClassLevelSetupFail.track_fail']['stderr']
profile_path = join(tmpdir, 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value']['profile'])
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split() == expected
# Calibration of iterations should not rerun setup
expected = ['setup']*2
assert times['time_examples.TimeWithRepeatCalibrate.time_it']['stderr'].split() == expected
def test_find_benchmarks(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = join(tmpdir, "env")
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
b = benchmarks.Benchmarks(conf, regex='secondary')
assert len(b) == 3
b = benchmarks.Benchmarks(conf, regex='example')
assert len(b) == 14
b = benchmarks.Benchmarks(conf, regex='time_example_benchmark_1')
assert len(b) == 2
b = benchmarks.Benchmarks(conf, regex=['time_example_benchmark_1',
'some regexp that does not match anything'])
assert len(b) == 2
b = benchmarks.Benchmarks(conf)
assert len(b) == 18
envs = list(environment.get_environments(conf))
b = benchmarks.Benchmarks(conf)
times = b.run_benchmarks(envs[0], profile=True, show_stderr=True)
assert len(times) == len(b)
assert times[
'time_examples.TimeSuite.time_example_benchmark_1']['result'] is not None
# Benchmarks that raise exceptions should have a time of "None"
assert times[
'time_secondary.TimeSecondary.time_exception']['result'] is None
assert times[
'subdir.time_subdir.time_foo']['result'] is not None
assert times[
'mem_examples.mem_list']['result'] > 2000
assert times[
'time_secondary.track_value']['result'] == 42.0
assert 'profile' in times[
'time_secondary.track_value']
assert 'stderr' in times[
'time_examples.time_with_warnings']
assert times['time_examples.time_with_warnings']['errcode'] != 0
assert times['params_examples.track_param']['result']['params'] == [["<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"]]
assert times['params_examples.track_param']['result']['result'] == [42, 42]
assert times['params_examples.mem_param']['result']['params'] == [['10', '20'], ['2', '3']]
assert len(times['params_examples.mem_param']['result']['result']) == 2*2
assert times['params_examples.ParamSuite.track_value']['result']['params'] == [["'a'", "'b'", "'c'"]]
assert times['params_examples.ParamSuite.track_value']['result']['result'] == [1+0, 2+0, 3+0]
assert isinstance(times['params_examples.TuningTest.time_it']['result']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result']['result'][1], float)
assert util.is_nan(times['params_examples.time_skip']['result']['result'][2])
assert times['peakmem_examples.peakmem_list']['result'] >= 4 * 2**20
profile_path = join(tmpdir, 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value']['profile'])
pstats.Stats(profile_path)
