def test_json_timestamp(tmpdir):
# Check that per-benchmark timestamps are saved as JS timestamps in the result file
tmpdir = six.text_type(tmpdir)
stamp0 = datetime.datetime(1970, 1, 1)
stamp1 = datetime.datetime(1971, 1, 1)
stamp2 = datetime.datetime.utcnow()
r = results.Results({'machine': 'mach'}, {}, 'aaaa',
util.datetime_to_timestamp(stamp0), 'py', 'env')
value = {
'result': [42],
'params': [],
'stats': None,
'samples': None,
'number': None,
'started_at': stamp1,
'ended_at': stamp2
}
r.add_result('some_benchmark', value)
r.save(tmpdir)
r = util.load_json(join(tmpdir, 'mach', 'aaaa-env.json'))
assert r['started_at']['some_benchmark'] == util.datetime_to_js_timestamp(
stamp1)
assert r['ended_at']['some_benchmark'] == util.datetime_to_js_timestamp(
stamp2)
def test_json_timestamp(tmpdir):
# Check that per-benchmark timestamps are saved as JS timestamps in the result file
tmpdir = six.text_type(tmpdir)
stamp0 = datetime.datetime(1970, 1, 1)
stamp1 = datetime.datetime(1971, 1, 1)
stamp2 = datetime.datetime.utcnow()
r = results.Results({'machine': 'mach'}, {}, 'aaaa',
util.datetime_to_timestamp(stamp0), 'py', 'env', {})
value = runner.BenchmarkResult(result=[42],
samples=[None],
number=[None],
profile=None,
errcode=0,
stderr='')
benchmark = {
'name': 'some_benchmark',
'version': 'some version',
'params': []
}
r.add_result(benchmark, value, started_at=stamp1, ended_at=stamp2)
r.save(tmpdir)
r = util.load_json(join(tmpdir, 'mach', 'aaaa-env.json'))
assert r['started_at']['some_benchmark'] == util.datetime_to_js_timestamp(
stamp1)
assert r['ended_at']['some_benchmark'] == util.datetime_to_js_timestamp(
stamp2)
def test_json_timestamp(tmpdir):
# Check that per-benchmark timestamps are saved as JS timestamps in the result file
tmpdir = six.text_type(tmpdir)
stamp0 = datetime.datetime(1970, 1, 1)
stamp1 = datetime.datetime(1971, 1, 1)
stamp2 = datetime.datetime.utcnow()
r = results.Results({'machine': 'mach'}, {}, 'aaaa', util.datetime_to_timestamp(stamp0),
'py', 'env')
value = runner.BenchmarkResult(
result=[42],
samples=[None],
number=[None],
profile=None,
errcode=0,
stderr=''
)
benchmark = {'name': 'some_benchmark', 'version': 'some version', 'params': []}
r.add_result(benchmark, value, started_at=stamp1, ended_at=stamp2)
r.save(tmpdir)
r = util.load_json(join(tmpdir, 'mach', 'aaaa-env.json'))
assert r['started_at']['some_benchmark'] == util.datetime_to_js_timestamp(stamp1)
assert r['ended_at']['some_benchmark'] == util.datetime_to_js_timestamp(stamp2)
def test_json_timestamp(tmpdir):
# Check that per-benchmark timestamps are saved as JS timestamps in the result file
tmpdir = six.text_type(tmpdir)
stamp0 = datetime.datetime(1970, 1, 1)
stamp1 = datetime.datetime(1971, 1, 1)
stamp2 = datetime.datetime.utcnow()
r = results.Results({'machine': 'mach'}, {}, 'aaaa', util.datetime_to_timestamp(stamp0),
'py', 'env')
value = {
'result': [42],
'params': [],
'stats': None,
'samples': None,
'number': None,
'started_at': stamp1,
'ended_at': stamp2
}
r.add_result('some_benchmark', value, "some version")
r.save(tmpdir)
r = util.load_json(join(tmpdir, 'mach', 'aaaa-env.json'))
assert r['started_at']['some_benchmark'] == util.datetime_to_js_timestamp(stamp1)
assert r['ended_at']['some_benchmark'] == util.datetime_to_js_timestamp(stamp2)
def test_json_timestamp(tmpdir):
# Check that per-benchmark timestamps are saved as JS timestamps in the result file
tmpdir = six.text_type(tmpdir)
stamp0 = datetime.datetime(1970, 1, 1)
stamp1 = datetime.datetime(1971, 1, 1)
stamp2 = datetime.datetime.utcnow()
r = results.Results({'machine': 'mach'}, {}, 'aaaa', util.datetime_to_timestamp(stamp0),
'py', 'env')
r.add_result('some_benchmark', 42, stamp1, stamp2)
r.save(tmpdir)
r = util.load_json(join(tmpdir, 'mach', 'aaaa-env.json'))
assert r['started_at']['some_benchmark'] == util.datetime_to_js_timestamp(stamp1)
assert r['ended_at']['some_benchmark'] == util.datetime_to_js_timestamp(stamp2)
def test_json_timestamp(tmpdir):
# Check that per-benchmark timestamps are saved as JS timestamps in the result file
tmpdir = str(tmpdir)
stamp0 = datetime.datetime(1970, 1, 1)
stamp1 = datetime.datetime(1971, 1, 1)
duration = 1.5
r = results.Results({'machine': 'mach'}, {}, 'aaaa',
util.datetime_to_timestamp(stamp0), 'py', 'env', {})
value = runner.BenchmarkResult(result=[42],
samples=[None],
number=[None],
profile=None,
errcode=0,
stderr='')
benchmark = {
'name': 'some_benchmark',
'version': 'some version',
'params': []
}
r.add_result(benchmark, value, started_at=stamp1, duration=duration)
r.save(tmpdir)
r = util.load_json(join(tmpdir, 'mach', 'aaaa-env.json'))
keys = r['result_columns']
values = dict(zip(keys, r['results']['some_benchmark']))
assert values['started_at'] == util.datetime_to_js_timestamp(stamp1)
assert values['duration'] == duration
def test_datetime_to_js_timestamp():
tss = [0, 0.5, -0.5, 12345.6789, -12345.6789, 1535910708.7767508]
for ts in tss:
t = datetime.datetime.utcfromtimestamp(ts)
ts2 = util.datetime_to_js_timestamp(t)
assert abs(ts * 1000 - ts2) <= 0.5
# Check sub-second precision
ms = 50
ts = datetime.datetime(1970, 1, 1, 0, 0, 0, 1000 * ms)
assert util.datetime_to_js_timestamp(ts) == ms
# Check rounding
ts = datetime.datetime(1970, 1, 1, 0, 0, 0, 500)
assert util.datetime_to_js_timestamp(ts) == 1
ts = datetime.datetime(1970, 1, 1, 0, 0, 0, 499)
assert util.datetime_to_js_timestamp(ts) == 0
def test_datetime_to_js_timestamp():
tss = [0, 0.5, -0.5, 12345.6789, -12345.6789,
1535910708.7767508]
for ts in tss:
t = datetime.datetime.utcfromtimestamp(ts)
ts2 = util.datetime_to_js_timestamp(t)
assert abs(ts * 1000 - ts2) <= 0.5
# Check sub-second precision
ms = 50
ts = datetime.datetime(1970, 1, 1, 0, 0, 0, 1000*ms)
assert util.datetime_to_js_timestamp(ts) == ms
# Check rounding
ts = datetime.datetime(1970, 1, 1, 0, 0, 0, 500)
assert util.datetime_to_js_timestamp(ts) == 1
ts = datetime.datetime(1970, 1, 1, 0, 0, 0, 499)
assert util.datetime_to_js_timestamp(ts) == 0
def test_run_build_failure(basic_conf):
tmpdir, local, conf, machine_file = basic_conf
conf.matrix = {}
# Add a commit that fails to build
dvcs = tools.Git(conf.repo)
setup_py = join(dvcs.path, 'setup.py')
with open(setup_py, 'r') as f:
setup_py_content = f.read()
with open(setup_py, 'w') as f:
f.write("assert False")
dvcs.add(join(dvcs.path, 'setup.py'))
dvcs.commit("Break setup.py")
with open(setup_py, 'w') as f:
f.write(setup_py_content)
dvcs.add(join(dvcs.path, 'setup.py'))
dvcs.commit("Fix setup.py")
# Test running it
timestamp = util.datetime_to_js_timestamp(datetime.datetime.utcnow())
bench_name = 'time_secondary.track_value'
for commit in ['master^!', 'master~1^!']:
tools.run_asv_with_conf(conf, 'run', commit,
'--quick', '--show-stderr',
'--bench', bench_name,
_machine_file=machine_file)
# Check results
hashes = dvcs.get_branch_hashes()
fn_broken, = glob.glob(join(tmpdir, 'results_workflow', 'orangutan',
hashes[1][:8] + '-*.json'))
fn_ok, = glob.glob(join(tmpdir, 'results_workflow', 'orangutan',
hashes[0][:8] + '-*.json'))
data_broken = util.load_json(fn_broken)
data_ok = util.load_json(fn_ok)
for data in (data_broken, data_ok):
value = dict(zip(data['result_columns'], data['results'][bench_name]))
assert value['started_at'] >= timestamp
if data is data_broken:
assert 'duration' not in value
else:
assert value['duration'] >= 0
assert len(data_broken['results']) == 1
assert len(data_ok['results']) == 1
assert data_broken['result_columns'][0] == 'result'
assert data_ok['result_columns'][0] == 'result'
assert data_broken['results'][bench_name][0] is None
assert data_ok['results'][bench_name][0] == [42.0]
# Check that parameters were also saved
assert data_broken['params'] == data_ok['params']
def test_run_build_failure(basic_conf):
tmpdir, local, conf, machine_file = basic_conf
conf.matrix = {}
# Add a commit that fails to build
dvcs = tools.Git(conf.repo)
setup_py = join(dvcs.path, 'setup.py')
with open(setup_py, 'r') as f:
setup_py_content = f.read()
with open(setup_py, 'w') as f:
f.write("assert False")
dvcs.add(join(dvcs.path, 'setup.py'))
dvcs.commit("Break setup.py")
with open(setup_py, 'w') as f:
f.write(setup_py_content)
dvcs.add(join(dvcs.path, 'setup.py'))
dvcs.commit("Fix setup.py")
# Test running it
timestamp = util.datetime_to_js_timestamp(datetime.datetime.utcnow())
bench_name = 'time_secondary.track_value'
for commit in ['master^!', 'master~1^!']:
tools.run_asv_with_conf(conf, 'run', commit,
'--quick', '--show-stderr',
'--bench', bench_name,
_machine_file=machine_file)
# Check results
hashes = dvcs.get_branch_hashes()
fn_broken, = glob.glob(join(tmpdir, 'results_workflow', 'orangutan',
hashes[1][:8] + '-*.json'))
fn_ok, = glob.glob(join(tmpdir, 'results_workflow', 'orangutan',
hashes[0][:8] + '-*.json'))
data_broken = util.load_json(fn_broken)
data_ok = util.load_json(fn_ok)
for data in (data_broken, data_ok):
assert data['started_at'][bench_name] >= timestamp
assert data['ended_at'][bench_name] >= data['started_at'][bench_name]
assert len(data_broken['results']) == 1
assert len(data_ok['results']) == 1
assert data_broken['results'][bench_name] is None
assert data_ok['results'][bench_name] == 42.0
# Check that parameters were also saved
assert data_broken['params'] == data_ok['params']
def test_run_benchmarks(benchmarks_fixture, tmpdir):
conf, repo, envs, commit_hash = benchmarks_fixture
start_timestamp = datetime.datetime.utcnow()
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
# Old results to append to
results = Results.unnamed()
name = 'time_examples.TimeSuite.time_example_benchmark_1'
results.add_result(b[name],
runner.BenchmarkResult(result=[1],
samples=[[42.0, 24.0]],
number=[1],
errcode=0,
stderr='',
profile=None),
record_samples=True)
# Run
runner.run_benchmarks(b,
envs[0],
results=results,
profile=True,
show_stderr=True,
append_samples=True,
record_samples=True)
times = ResultsWrapper(results, b)
end_timestamp = datetime.datetime.utcnow()
assert len(times) == len(b)
assert times[
'time_examples.TimeSuite.time_example_benchmark_1'].result != [None]
stats = results.get_result_stats(name, b[name]['params'])
assert isinstance(stats[0]['std'], float)
# The exact number of samples may vary if the calibration is not fully accurate
samples = results.get_result_samples(name, b[name]['params'])
assert len(samples[0]) >= 4
# Explicitly provided 'prev_samples` should come first
assert samples[0][:2] == [42.0, 24.0]
# Benchmarks that raise exceptions should have a time of "None"
assert times['time_secondary.TimeSecondary.time_exception'].result == [
None
]
assert times['subdir.time_subdir.time_foo'].result != [None]
if not ON_PYPY:
# XXX: the memory benchmarks don't work on Pypy, since asizeof
# is CPython-only
assert times['mem_examples.mem_list'].result[0] > 1000
assert times['time_secondary.track_value'].result == [42.0]
assert times['time_secondary.track_value'].profile is not None
assert isinstance(times['time_examples.time_with_warnings'].stderr,
type(''))
assert times['time_examples.time_with_warnings'].errcode != 0
assert times['time_examples.TimeWithBadTimer.time_it'].result == [0.0]
assert times['params_examples.track_param'].params == [[
"<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"
]]
assert times['params_examples.track_param'].result == [42, 42]
assert times['params_examples.mem_param'].params == [['10', '20'],
['2', '3']]
assert len(times['params_examples.mem_param'].result) == 2 * 2
assert times['params_examples.ParamSuite.track_value'].params == [[
"'a'", "'b'", "'c'"
]]
assert times['params_examples.ParamSuite.track_value'].result == [
1 + 0, 2 + 0, 3 + 0
]
assert isinstance(times['params_examples.TuningTest.time_it'].result[0],
float)
assert isinstance(times['params_examples.TuningTest.time_it'].result[1],
float)
assert isinstance(times['params_examples.time_skip'].result[0], float)
assert isinstance(times['params_examples.time_skip'].result[1], float)
assert util.is_nan(times['params_examples.time_skip'].result[2])
assert times['peakmem_examples.peakmem_list'].result[0] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example'].result == [
500
]
assert times['cache_examples.ClassLevelSetup.track_example2'].result == [
500
]
assert times['cache_examples.track_cache_foo'].result == [42]
assert times['cache_examples.track_cache_bar'].result == [12]
assert times['cache_examples.track_my_cache_foo'].result == [0]
assert times['cache_examples.ClassLevelSetupFail.track_fail'].result == [
None
]
assert 'raise RuntimeError()' in times[
'cache_examples.ClassLevelSetupFail.track_fail'].stderr
assert times[
'cache_examples.ClassLevelCacheTimeout.track_fail'].result == [None]
assert times[
'cache_examples.ClassLevelCacheTimeoutSuccess.track_success'].result == [
0
]
assert times['cache_examples.time_fail_second_run'].result == [None]
assert times['cache_examples.time_fail_second_run'].samples == [None]
profile_path = join(six.text_type(tmpdir), 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value'].profile)
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it'].stderr.split(
) == expected
# Calibration of iterations should not rerun setup
expected = (['setup'] * 2, ['setup'] * 3)
assert times['time_examples.TimeWithRepeatCalibrate.time_it'].stderr.split(
) in expected
# Check tuple-form repeat attribute produced results
assert 2 <= len(times['time_examples.time_auto_repeat'].samples[0]) <= 4
# Check run time timestamps
for name, result in times.items():
assert result.started_at >= util.datetime_to_js_timestamp(
start_timestamp)
assert result.ended_at >= result.started_at
assert result.ended_at <= util.datetime_to_js_timestamp(end_timestamp)
