def _test_generic_repo(conf,
tmpdir,
hash_range,
master,
branch,
is_remote=False):
workcopy_dir = tempfile.mkdtemp(dir=tmpdir, prefix="workcopy")
os.rmdir(workcopy_dir)
# check mirroring fails early if *mirror_dir* exists but is not
# a mirror
if is_remote:
if os.path.isdir(conf.project):
shutil.rmtree(conf.project)
os.makedirs(join(conf.project, 'hello'))
with pytest.raises(util.UserError):
r = repo.get_repo(conf)
shutil.rmtree(conf.project)
# basic checkouts
r = repo.get_repo(conf)
# Subrepo creation
r.checkout(workcopy_dir, master)
assert os.path.exists(join(workcopy_dir, "setup.py"))
for filename in ("README", "untracked"):
with open(join(workcopy_dir, filename), "wb") as fd:
fd.write(b"foo")
# After checkout the subrepo has been cleaned
r.checkout(workcopy_dir, branch)
assert not os.path.exists(join(workcopy_dir, "untracked"))
with open(join(workcopy_dir, "README"), "rb") as fd:
data = fd.read(33)
assert data == b"This is the asv_test_repo project"
r.checkout(workcopy_dir, master)
# check recovering from corruption
for pth in ['.hg', '.git']:
pth = os.path.join(workcopy_dir, pth)
if os.path.isdir(pth):
shutil.rmtree(pth)
r.checkout(workcopy_dir, master)
hashes = r.get_hashes_from_range(hash_range)
assert len(hashes) == 4
dates = [r.get_date(hash) for hash in hashes]
assert dates == sorted(dates)[::-1]
tags = r.get_tags()
for tag in tags:
r.get_date_from_name(tag)
def _test_generic_repo(conf, tmpdir, hash_range, master, branch, is_remote=False):
workcopy_dir = tempfile.mkdtemp(dir=tmpdir, prefix="workcopy")
os.rmdir(workcopy_dir)
# check mirroring fails early if *mirror_dir* exists but is not
# a mirror
if is_remote:
if os.path.isdir(conf.project):
shutil.rmtree(conf.project)
os.makedirs(join(conf.project, 'hello'))
with pytest.raises(util.UserError):
r = repo.get_repo(conf)
shutil.rmtree(conf.project)
# basic checkouts
r = repo.get_repo(conf)
# Subrepo creation
r.checkout(workcopy_dir, master)
assert os.path.exists(join(workcopy_dir, "setup.py"))
for filename in ("README", "untracked"):
with open(join(workcopy_dir, filename), "wb") as fd:
fd.write(b"foo")
# After checkout the subrepo has been cleaned
r.checkout(workcopy_dir, branch)
assert not os.path.exists(join(workcopy_dir, "untracked"))
with open(join(workcopy_dir, "README"), "rb") as fd:
data = fd.read(33)
assert data == b"This is the asv_test_repo project"
r.checkout(workcopy_dir, master)
# check recovering from corruption
for pth in ['.hg', '.git']:
pth = os.path.join(workcopy_dir, pth)
if os.path.isdir(pth):
shutil.rmtree(pth)
r.checkout(workcopy_dir, master)
hashes = r.get_hashes_from_range(hash_range)
assert len(hashes) == 4
dates = [r.get_date(hash) for hash in hashes]
assert dates == sorted(dates)[::-1]
tags = r.get_tags()
for tag in tags:
r.get_date_from_name(tag)
def generate_result_dir(tmpdir, dvcs, values, branches=None):
result_dir = join(tmpdir, "results")
os.makedirs(result_dir)
html_dir = join(tmpdir, "html")
machine_dir = join(result_dir, "tarzan")
os.makedirs(machine_dir)
if branches is None:
branches = [None]
conf = config.Config.from_json({
'results_dir': result_dir,
'html_dir': html_dir,
'repo': dvcs.path,
'project': 'asv',
'branches': branches or [None],
})
repo = get_repo(conf)
util.write_json(join(machine_dir, "machine.json"), {
'machine': 'tarzan',
'version': 1,
})
timestamp = datetime.datetime.utcnow()
benchmark_version = sha256(os.urandom(16)).hexdigest()
params = None
param_names = None
for commit, value in values.items():
if isinstance(value, dict):
params = value["params"]
result = Results({"machine": "tarzan"}, {}, commit,
repo.get_date_from_name(commit), "2.7", None)
value = {
'result': [value],
'params': [],
'started_at': timestamp,
'ended_at': timestamp,
'stats': None,
'samples': None,
'number': None,
}
result.add_result("time_func", value, benchmark_version)
result.save(result_dir)
if params:
param_names = ["param{}".format(k) for k in range(len(params))]
util.write_json(join(result_dir, "benchmarks.json"), {
"time_func": {
"name": "time_func",
"params": params or [],
"param_names": param_names or [],
"version": benchmark_version,
}
},
api_version=1)
return conf
def test_filter_date_period(tmpdir, dvcs_type):
tmpdir = six.text_type(tmpdir)
dates = [
datetime.datetime(2001, 1, 1),
datetime.datetime(2001, 1, 2),
datetime.datetime(2001, 1, 8)
]
dvcs = tools.generate_repo_from_ops(
tmpdir, dvcs_type,
[("commit", j, dates[j]) for j in range(len(dates))])
commits = dvcs.get_branch_hashes()[::-1]
assert len(commits) == len(dates)
conf = config.Config()
conf.dvcs = dvcs_type
conf.repo = dvcs.path
r = repo.get_repo(conf)
# Basic filtering
weekly_commits = r.filter_date_period(commits, 60*60*24*7)
assert weekly_commits == [commits[0], commits[2]]
daily_commits = r.filter_date_period(commits, 60*60*24)
assert daily_commits == commits
# Test with old_commits specified
monthly_commits = r.filter_date_period(commits[1:], 60*60*24*30, commits[:1])
assert monthly_commits == []
def test_conf_inside_benchmarks_dir(tmpdir):
# Test that the configuration file can be inside the benchmark suite
tmpdir = six.text_type(tmpdir)
benchmark_dir = os.path.join(tmpdir, 'benchmark')
os.makedirs(benchmark_dir)
with open(os.path.join(benchmark_dir, '__init__.py'), 'w') as f:
# Test also benchmark in top-level __init__.py
f.write("def track_this(): pass")
with open(os.path.join(benchmark_dir, 'bench.py'), 'w') as f:
f.write("def track_this(): pass")
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = '.'
d['repo'] = tools.generate_test_repo(tmpdir, [[0, 1]]).path
conf = config.Config.from_json(d)
# NB. conf_dir == getcwd()
os.chdir(benchmark_dir)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf,
repo,
envs, [commit_hash],
regex='track_this')
assert set(b.keys()) == {'track_this', 'bench.track_this'}
def test_quick(tmpdir):
# Check that the quick option works
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
b = benchmarks.Benchmarks(conf, repo, envs)
skip_names = [
name for name in b.keys()
if name != 'time_examples.TimeWithRepeat.time_it'
]
times = b.run_benchmarks(envs[0],
quick=True,
show_stderr=True,
skip=skip_names)
assert len(times) == 1
# Check that the benchmark was run only once. The result for quick==False
# is tested above in test_find_benchmarks
expected = ["<1>"]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split(
) == expected
def test_quick(tmpdir):
# Check that the quick option works
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
skip_names = [name for name in b.keys() if name != 'time_examples.TimeWithRepeat.time_it']
times = b.run_benchmarks(envs[0], quick=True, show_stderr=True, skip=skip_names)
assert len(times) == 1
# Check that the benchmark was run only once. The result for quick==False
# is tested above in test_find_benchmarks
expected = ["<1>"]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split() == expected
def test_root_ceiling(dvcs_type, tmpdir):
# Check that git/hg does not try to look for repository in parent
# directories.
tmpdir = six.text_type(tmpdir)
dvcs1 = tools.generate_repo_from_ops(tmpdir, dvcs_type, [("commit", 1)])
dvcs2 = tools.generate_repo_from_ops(tmpdir, dvcs_type, [("commit", 2)])
commit1 = dvcs1.get_branch_hashes()[0]
commit2 = dvcs2.get_branch_hashes()[0]
conf = config.Config()
conf.branches = []
conf.dvcs = dvcs_type
conf.project = join(tmpdir, "repo")
conf.repo = dvcs1.path
r = repo.get_repo(conf)
# Checkout into a subdir inside another repository
workcopy_dir = join(dvcs2.path, "workcopy")
r.checkout(workcopy_dir, commit1)
# Corrupt the checkout
for pth in ['.hg', '.git']:
pth = os.path.join(workcopy_dir, pth)
if os.path.isdir(pth):
shutil.rmtree(pth)
# Operation must fail (commit2 is not in dvcs1), not use the
# parent repository
with pytest.raises(Exception):
r.checkout(workcopy_dir, commit2)
def test_no_such_name_error(dvcs_type, tmpdir):
tmpdir = six.text_type(tmpdir)
dvcs = tools.generate_test_repo(tmpdir, values=[0], dvcs_type=dvcs_type)
conf = config.Config()
conf.branches = []
conf.dvcs = dvcs_type
conf.project = "project"
conf.repo = dvcs.path
r = repo.get_repo(conf)
# Check that NoSuchNameError error gets raised correctly
assert r.get_hash_from_name(None) == dvcs.get_hash(r._default_branch)
with pytest.raises(repo.NoSuchNameError):
r.get_hash_from_name("badbranch")
if dvcs_type == "git":
# Corrupted repository/etc should not give NoSuchNameError
util.long_path_rmtree(join(dvcs.path, ".git"))
with pytest.raises(Exception) as excinfo:
r.get_hash_from_name(None)
assert excinfo.type not in (AssertionError, repo.NoSuchNameError)
elif dvcs_type == "hg":
# hglib seems to do some caching, so this doesn't work
pass
def test_root_ceiling(dvcs_type, tmpdir):
# Check that git/hg does not try to look for repository in parent
# directories.
tmpdir = six.text_type(tmpdir)
dvcs1 = tools.generate_repo_from_ops(tmpdir, dvcs_type, [("commit", 1)])
dvcs2 = tools.generate_repo_from_ops(tmpdir, dvcs_type, [("commit", 2)])
commit1 = dvcs1.get_branch_hashes()[0]
commit2 = dvcs2.get_branch_hashes()[0]
conf = config.Config()
conf.branches = []
conf.dvcs = dvcs_type
conf.project = join(tmpdir, "repo")
conf.repo = dvcs1.path
r = repo.get_repo(conf)
# Checkout into a subdir inside another repository
workcopy_dir = join(dvcs2.path, "workcopy")
r.checkout(workcopy_dir, commit1)
# Corrupt the checkout
for pth in ['.hg', '.git']:
pth = os.path.join(workcopy_dir, pth)
if os.path.isdir(pth):
shutil.rmtree(pth)
# Operation must fail (commit2 is not in dvcs1), not use the
# parent repository
with pytest.raises(Exception):
r.checkout(workcopy_dir, commit2)
def test_no_such_name_error(dvcs_type, tmpdir):
tmpdir = six.text_type(tmpdir)
dvcs = tools.generate_test_repo(tmpdir, values=[0], dvcs_type=dvcs_type)
conf = config.Config()
conf.branches = []
conf.dvcs = dvcs_type
conf.project = "project"
conf.repo = dvcs.path
r = repo.get_repo(conf)
# Check that NoSuchNameError error gets raised correctly
assert r.get_hash_from_name(None) == dvcs.get_hash(r._default_branch)
with pytest.raises(repo.NoSuchNameError):
r.get_hash_from_name("badbranch")
if dvcs_type == "git":
# Corrupted repository/etc should not give NoSuchNameError
util.long_path_rmtree(join(dvcs.path, ".git"))
with pytest.raises(Exception) as excinfo:
r.get_hash_from_name(None)
assert excinfo.type not in (AssertionError, repo.NoSuchNameError)
elif dvcs_type == "hg":
# hglib seems to do some caching, so this doesn't work
pass
def test_install_env_matrix_values(tmpdir):
tmpdir = six.text_type(tmpdir)
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
commit_hash = dvcs.get_branch_hashes()[0]
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.repo = os.path.abspath(dvcs.path)
conf.matrix = {
'env': {
'SOME_ASV_TEST_BUILD_VALUE': '1'
},
'env_nobuild': {
'SOME_ASV_TEST_NON_BUILD_VALUE': '1'
}
}
repo = get_repo(conf)
env = list(environment.get_environments(conf, None))[0]
env.create()
env.install_project(conf, repo, commit_hash)
env.run([
'-c', 'import asv_test_repo.build_time_env as t, sys; '
'sys.exit(0 if t.env["SOME_ASV_TEST_BUILD_VALUE"] == "1" else 1)'
])
env.run([
'-c', 'import asv_test_repo.build_time_env as t, sys; '
'sys.exit(0 if "SOME_ASV_TEST_NON_BUILD_VALUE" not in t.env else 1)'
])
def test_conf_inside_benchmarks_dir(tmpdir):
# Test that the configuration file can be inside the benchmark suite
tmpdir = six.text_type(tmpdir)
benchmark_dir = os.path.join(tmpdir, 'benchmark')
os.makedirs(benchmark_dir)
with open(os.path.join(benchmark_dir, '__init__.py'), 'w') as f:
# Test also benchmark in top-level __init__.py
f.write("def track_this(): pass")
with open(os.path.join(benchmark_dir, 'bench.py'), 'w') as f:
f.write("def track_this(): pass")
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = '.'
d['repo'] = tools.generate_test_repo(tmpdir, [[0, 1]]).path
conf = config.Config.from_json(d)
# NB. conf_dir == getcwd()
os.chdir(benchmark_dir)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='track_this')
assert set(b.keys()) == {'track_this', 'bench.track_this'}
def test_build_isolation(tmpdir):
# build should not fail with build_cache on projects that have pyproject.toml
tmpdir = six.text_type(tmpdir)
# Create installable repository with pyproject.toml in it
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
fn = os.path.join(dvcs.path, 'pyproject.toml')
with open(fn, 'w') as f:
f.write('[build-system]\n' 'requires = ["wheel", "setuptools"]')
dvcs.add(fn)
dvcs.commit("Add pyproject.toml")
commit_hash = dvcs.get_hash("master")
# Setup config
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.matrix = {}
conf.repo = os.path.abspath(dvcs.path)
conf.build_cache_size = 8
repo = get_repo(conf)
env = list(environment.get_environments(conf, None))[0]
env.create()
# Project installation should succeed
env.install_project(conf, repo, commit_hash)
def test_build_isolation(tmpdir):
# build should not fail with build_cache on projects that have pyproject.toml
tmpdir = six.text_type(tmpdir)
# Create installable repository with pyproject.toml in it
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
fn = os.path.join(dvcs.path, 'pyproject.toml')
with open(fn, 'w') as f:
f.write('[build-system]\n'
'requires = ["wheel", "setuptools"]')
dvcs.add(fn)
dvcs.commit("Add pyproject.toml")
commit_hash = dvcs.get_hash("master")
# Setup config
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.matrix = {}
conf.repo = os.path.abspath(dvcs.path)
conf.build_cache_size = 8
repo = get_repo(conf)
env = list(environment.get_environments(conf, None))[0]
env.create()
# Project installation should succeed
env.install_project(conf, repo, commit_hash)
def test_install_success(tmpdir):
# Check that install_project really installs the package. (gh-805)
# This may fail if pip in install_command e.g. gets confused by an .egg-info
# directory in its cwd to think the package is already installed.
tmpdir = six.text_type(tmpdir)
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
commit_hash = dvcs.get_branch_hashes()[0]
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.repo = os.path.abspath(dvcs.path)
conf.matrix = {}
conf.build_cache_size = 0
repo = get_repo(conf)
env = list(environment.get_environments(conf, None))[0]
env.create()
env.install_project(conf, repo, commit_hash)
env.run([
'-c',
'import asv_test_repo as t, sys; sys.exit(0 if t.dummy_value == 0 else 1)'
])
def generate_result_dir(tmpdir, dvcs, values, branches=None):
result_dir = join(tmpdir, "results")
os.makedirs(result_dir)
html_dir = join(tmpdir, "html")
machine_dir = join(result_dir, "tarzan")
os.makedirs(machine_dir)
if branches is None:
branches = [None]
conf = config.Config.from_json({
'results_dir': result_dir,
'html_dir': html_dir,
'repo': dvcs.path,
'project': 'asv',
'branches': branches or [None],
})
repo = get_repo(conf)
util.write_json(join(machine_dir, "machine.json"), {
'machine': 'tarzan',
'version': 1,
})
timestamp = datetime.datetime.utcnow()
benchmark_version = sha256(os.urandom(16)).hexdigest()
params = None
param_names = None
for commit, value in values.items():
if isinstance(value, dict):
params = value["params"]
result = Results({"machine": "tarzan"}, {}, commit,
repo.get_date_from_name(commit), "2.7", None)
value = {
'result': [value],
'params': [],
'started_at': timestamp,
'ended_at': timestamp,
'stats': None,
'samples': None,
'number': None,
}
result.add_result("time_func", value, benchmark_version)
result.save(result_dir)
if params:
param_names = ["param{}".format(k) for k in range(len(params))]
util.write_json(join(result_dir, "benchmarks.json"), {
"time_func": {
"name": "time_func",
"params": params or [],
"param_names": param_names or [],
"version": benchmark_version,
}
}, api_version=1)
return conf
def _test_branches(conf, branch_commits):
r = repo.get_repo(conf)
assert len(conf.branches) == 2
for branch in conf.branches:
commits = r.get_branch_commits(branch)
for commit in branch_commits[branch]:
assert commit in commits
def _test_branches(conf, branch_commits):
r = repo.get_repo(conf)
assert len(conf.branches) == 2
for branch in conf.branches:
commits = r.get_branch_commits(branch)
for commit in branch_commits[branch]:
assert commit in commits
def _test_generic_repo(conf,
tmpdir,
hash_range,
master,
branch,
is_remote=False):
workcopy_dir = tempfile.mkdtemp(dir=tmpdir, prefix="workcopy")
os.rmdir(workcopy_dir)
# check mirroring fails early if *mirror_dir* exists but is not
# a mirror
if is_remote:
if os.path.isdir(conf.project):
shutil.rmtree(conf.project)
os.makedirs(join(conf.project, 'hello'))
with pytest.raises(util.UserError):
r = repo.get_repo(conf)
shutil.rmtree(conf.project)
# basic checkouts
r = repo.get_repo(conf)
r.checkout(workcopy_dir, master)
r.checkout(workcopy_dir, branch)
r.checkout(workcopy_dir, master)
# check recovering from corruption
for pth in ['.hg', '.git']:
pth = os.path.join(workcopy_dir, pth)
if os.path.isdir(pth):
shutil.rmtree(pth)
r.checkout(workcopy_dir, master)
hashes = r.get_hashes_from_range(hash_range)
assert len(hashes) == 4
dates = [r.get_date(hash) for hash in hashes]
assert dates == sorted(dates)[::-1]
tags = r.get_tags()
for tag in tags:
r.get_date_from_name(tag)
def _generate_result_dir(values, commits_without_result=None):
dvcs = tools.generate_repo_from_ops(
tmpdir, dvcs_type, [("commit", i) for i in range(len(values))])
commits = list(reversed(dvcs.get_branch_hashes()))
commit_values = {}
commits_without_result = [commits[i] for i in commits_without_result or []]
for commit, value in zip(commits, values):
if commit not in commits_without_result:
commit_values[commit] = value
conf = tools.generate_result_dir(tmpdir, dvcs, commit_values)
repo = get_repo(conf)
return conf, repo, commits
def _generate_result_dir(values, commits_without_result=None):
dvcs = tools.generate_repo_from_ops(
tmpdir, dvcs_type, [("commit", i) for i in range(len(values))])
commits = list(reversed(dvcs.get_branch_hashes()))
commit_values = {}
commits_without_result = [commits[i] for i in commits_without_result or []]
for commit, value in zip(commits, values):
if commit not in commits_without_result:
commit_values[commit] = value
conf = tools.generate_result_dir(tmpdir, dvcs, commit_values)
repo = get_repo(conf)
return conf, repo, commits
def test_regression_multiple_branches(dvcs_type, tmpdir):
tmpdir = six.text_type(tmpdir)
if dvcs_type == "git":
master = "master"
elif dvcs_type == "hg":
master = "default"
dvcs = tools.generate_repo_from_ops(
tmpdir,
dvcs_type,
[
("commit", 1),
("checkout", "stable", master),
("commit", 1),
("checkout", master),
] + 4 * [
("commit", 1),
("checkout", "stable"),
("commit", 1),
("checkout", master),
] + 5 * [
("commit", 1),
("checkout", "stable"),
("commit", 2),
("checkout", master),
],
)
commit_values = {}
branches = dict((branch, list(reversed(dvcs.get_branch_hashes(branch))))
for branch in (master, "stable"))
for branch, values in (
(master, 10 * [1]),
("stable", 5 * [1] + 5 * [2]),
):
for commit, value in zip(branches[branch], values):
commit_values[commit] = value
conf = tools.generate_result_dir(tmpdir, dvcs, commit_values)
conf.branches = [master, "stable"]
tools.run_asv_with_conf(conf, "publish")
repo = get_repo(conf)
regressions = util.load_json(join(conf.html_dir, "regressions.json"))
graph_path = join('graphs', 'branch-stable', 'machine-tarzan',
'time_func.json')
# Regression occur on 5th commit of stable branch
revision = repo.get_revisions(commit_values.keys())[branches["stable"][5]]
expected = {
'regressions': [[
'time_func', graph_path, {
'branch': 'stable'
}, None, 2.0, 1.0, [[None, revision, 1.0, 2.0]]
]]
}
assert regressions == expected
def test_repo_git_annotated_tag_date(tmpdir):
tmpdir = six.text_type(tmpdir)
dvcs = tools.generate_test_repo(tmpdir, list(range(5)), dvcs_type='git')
conf = config.Config()
conf.project = 'sometest'
conf.repo = dvcs.path
r = repo.get_repo(conf)
d1 = r.get_date('tag1')
d2 = r.get_date(r.get_hash_from_name('tag1'))
assert d1 == d2
def test_repo_git_annotated_tag_date(tmpdir):
tmpdir = six.text_type(tmpdir)
dvcs = tools.generate_test_repo(tmpdir, list(range(5)), dvcs_type='git')
conf = config.Config()
conf.project = 'sometest'
conf.repo = dvcs.path
r = repo.get_repo(conf)
d1 = r.get_date('tag1')
d2 = r.get_date(r.get_hash_from_name('tag1'))
assert d1 == d2
def two_branch_repo_case(request, tmpdir):
r"""
This test ensure we follow the first parent in case of merges
The revision graph looks like this:
@ Revision 6 (default)
|
| o Revision 5 (stable)
| |
| o Merge master
|/|
o | Revision 4
| |
o | Merge stable
|\|
o | Revision 3
| |
| o Revision 2
|/
o Revision 1
"""
dvcs_type = request.param
tmpdir = six.text_type(tmpdir)
if dvcs_type == "git":
master = "master"
elif dvcs_type == "hg":
master = "default"
dvcs = tools.generate_repo_from_ops(tmpdir, dvcs_type, [
("commit", 1),
("checkout", "stable", master),
("commit", 2),
("checkout", master),
("commit", 3),
("merge", "stable"),
("commit", 4),
("checkout", "stable"),
("merge", master, "Merge master"),
("commit", 5),
("checkout", master),
("commit", 6),
])
conf = config.Config()
conf.branches = [master, "stable"]
conf.repo = dvcs.path
conf.project = join(tmpdir, "repo")
r = repo.get_repo(conf)
return dvcs, master, r, conf
def two_branch_repo_case(request, tmpdir):
r"""
This test ensure we follow the first parent in case of merges
The revision graph looks like this:
@ Revision 6 (default)
|
| o Revision 5 (stable)
| |
| o Merge master
|/|
o | Revision 4
| |
o | Merge stable
|\|
o | Revision 3
| |
| o Revision 2
|/
o Revision 1
"""
dvcs_type = request.param
tmpdir = six.text_type(tmpdir)
if dvcs_type == "git":
master = "master"
elif dvcs_type == "hg":
master = "default"
dvcs = tools.generate_repo_from_ops(tmpdir, dvcs_type, [
("commit", 1),
("checkout", "stable", master),
("commit", 2),
("checkout", master),
("commit", 3),
("merge", "stable"),
("commit", 4),
("checkout", "stable"),
("merge", master, "Merge master"),
("commit", 5),
("checkout", master),
("commit", 6),
])
conf = config.Config()
conf.branches = [master, "stable"]
conf.repo = dvcs.path
conf.project = join(tmpdir, "repo")
r = repo.get_repo(conf)
return dvcs, master, r, conf
def _test_generic_repo(conf, tmpdir, hash_range, master, branch, is_remote=False):
workcopy_dir = tempfile.mkdtemp(dir=tmpdir, prefix="workcopy")
os.rmdir(workcopy_dir)
# check mirroring fails early if *mirror_dir* exists but is not
# a mirror
if is_remote:
if os.path.isdir(conf.project):
shutil.rmtree(conf.project)
os.makedirs(join(conf.project, 'hello'))
with pytest.raises(util.UserError):
r = repo.get_repo(conf)
shutil.rmtree(conf.project)
# basic checkouts
r = repo.get_repo(conf)
r.checkout(workcopy_dir, master)
r.checkout(workcopy_dir, branch)
r.checkout(workcopy_dir, master)
# check recovering from corruption
for pth in ['.hg', '.git']:
pth = os.path.join(workcopy_dir, pth)
if os.path.isdir(pth):
shutil.rmtree(pth)
r.checkout(workcopy_dir, master)
hashes = r.get_hashes_from_range(hash_range)
assert len(hashes) == 4
dates = [r.get_date(hash) for hash in hashes]
assert dates == sorted(dates)[::-1]
tags = r.get_tags()
for tag in tags:
r.get_date_from_name(tag)
def test_set_commit_hash(capsys, existing_env_conf):
tmpdir, local, conf, machine_file = existing_env_conf
r = repo.get_repo(conf)
commit_hash = r.get_hash_from_name(r.get_branch_name())
tools.run_asv_with_conf(conf, 'run', '--set-commit-hash=' + commit_hash, _machine_file=join(tmpdir, 'asv-machine.json'))
env_name = list(environment.get_environments(conf, None))[0].name
result_filename = commit_hash[:conf.hash_length] + '-' + env_name + '.json'
assert result_filename in os.listdir(join('results_workflow', 'orangutan'))
result_path = join('results_workflow', 'orangutan', result_filename)
times = results.Results.load(result_path)
assert times.commit_hash == commit_hash
def test_set_commit_hash(capsys, existing_env_conf):
tmpdir, local, conf, machine_file = existing_env_conf
r = repo.get_repo(conf)
commit_hash = r.get_hash_from_name(r.get_branch_name())
tools.run_asv_with_conf(conf, 'run', '--set-commit-hash=' + commit_hash, _machine_file=join(tmpdir, 'asv-machine.json'))
env_name = list(environment.get_environments(conf, None))[0].name
result_filename = commit_hash[:conf.hash_length] + '-' + env_name + '.json'
assert result_filename in os.listdir(join('results_workflow', 'orangutan'))
result_path = join('results_workflow', 'orangutan', result_filename)
times = results.Results.load(result_path)
assert times.commit_hash == commit_hash
def test_invalid_benchmark_tree(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
d = {}
d.update(ASV_CONF_JSON)
d['benchmark_dir'] = INVALID_BENCHMARK_DIR
d['env_dir'] = "env"
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
with pytest.raises(util.UserError):
b = benchmarks.Benchmarks(conf, repo, envs)
def test_invalid_benchmark_tree(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
d = {}
d.update(ASV_CONF_JSON)
d['benchmark_dir'] = INVALID_BENCHMARK_DIR
d['env_dir'] = "env"
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
with pytest.raises(util.UserError):
b = benchmarks.Benchmarks(conf, repo, envs)
def _test_branches(conf, branch_commits, require_describe=False):
r = repo.get_repo(conf)
assert len(conf.branches) == 2
for branch in conf.branches:
commits = r.get_branch_commits(branch)
for commit in branch_commits[branch]:
assert commit in commits
name = r.get_name_from_hash(commit)
if require_describe:
assert name is not None
if name is not None:
assert r.get_hash_from_name(name) == commit
assert name in r.get_decorated_hash(commit)
def test_regression_multiple_branches(dvcs_type, tmpdir):
tmpdir = six.text_type(tmpdir)
if dvcs_type == "git":
master = "master"
elif dvcs_type == "hg":
master = "default"
dvcs = tools.generate_repo_from_ops(
tmpdir, dvcs_type, [
("commit", 1),
("checkout", "stable", master),
("commit", 1),
("checkout", master),
] + 4 * [
("commit", 1),
("checkout", "stable"),
("commit", 1),
("checkout", master),
] + 5 * [
("commit", 1),
("checkout", "stable"),
("commit", 2),
("checkout", master),
],
)
commit_values = {}
branches = dict(
(branch, list(reversed(dvcs.get_branch_hashes(branch))))
for branch in (master, "stable")
)
for branch, values in (
(master, 10 * [1]),
("stable", 5 * [1] + 5 * [2]),
):
for commit, value in zip(branches[branch], values):
commit_values[commit] = value
conf = tools.generate_result_dir(tmpdir, dvcs, commit_values)
conf.branches = [master, "stable"]
tools.run_asv_with_conf(conf, "publish")
repo = get_repo(conf)
regressions = util.load_json(join(conf.html_dir, "regressions.json"))
graph_path = join('graphs', 'branch-stable', 'machine-tarzan', 'time_func.json')
# Regression occur on 5th commit of stable branch
revision = repo.get_revisions(commit_values.keys())[branches["stable"][5]]
expected = {'regressions': [['time_func', graph_path, {'branch': 'stable'}, None,
[[[None, revision, 1.0, 2.0]], 2.0, 1.0]]]}
assert regressions == expected
def _test_branches(conf, branch_commits, require_describe=False):
r = repo.get_repo(conf)
assert len(conf.branches) == 2
for branch in conf.branches:
commits = r.get_branch_commits(branch)
for commit in branch_commits[branch]:
assert commit in commits
name = r.get_name_from_hash(commit)
if require_describe:
assert name is not None
if name is not None:
assert r.get_hash_from_name(name) == commit
assert name in r.get_decorated_hash(commit)
def test_installed_commit_hash(tmpdir):
tmpdir = six.text_type(tmpdir)
tmpdir = six.text_type(tmpdir)
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
commit_hash = dvcs.get_branch_hashes()[0]
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.repo = os.path.abspath(dvcs.path)
conf.matrix = {}
conf.build_cache_size = 0
repo = get_repo(conf)
def get_env():
return list(environment.get_environments(conf, None))[0]
env = get_env()
env.create()
# Check updating installed_commit_hash
assert env.installed_commit_hash == None
assert env._env_vars.get('ASV_COMMIT') == None
env.install_project(conf, repo, commit_hash)
assert env.installed_commit_hash == commit_hash
assert env._env_vars.get('ASV_COMMIT') == commit_hash
env = get_env()
assert env.installed_commit_hash == commit_hash
assert env._env_vars.get('ASV_COMMIT') == commit_hash
# Configuration change results to reinstall
env._project = "something"
assert env.installed_commit_hash == None
# Uninstall resets hash (but not ASV_COMMIT)
env = get_env()
env._uninstall_project()
assert env.installed_commit_hash == None
assert env._env_vars.get('ASV_COMMIT') != None
env = get_env()
assert env.installed_commit_hash == None
assert env._env_vars.get('ASV_COMMIT') == None
def _test_branches(conf, branch_commits):
r = repo.get_repo(conf)
branch_cache = BranchCache(conf, r)
assert len(conf.branches) == 2
commit_branches = {}
for branch in conf.branches:
commits = branch_cache.get_branch_commits(branch)
for commit in branch_commits[branch]:
assert commit in commits
commit_branches[commit] = branch
for commit, branch in commit_branches.items():
assert branch in branch_cache.get_branches(commit)
def test_invalid_benchmark_tree(tmpdir):
tmpdir = str(tmpdir)
os.chdir(tmpdir)
d = {}
d.update(ASV_CONF_JSON)
d['benchmark_dir'] = INVALID_BENCHMARK_DIR
d['env_dir'] = "env"
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
with pytest.raises(util.UserError):
benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
def test_installed_commit_hash(tmpdir):
tmpdir = six.text_type(tmpdir)
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
commit_hash = dvcs.get_branch_hashes()[0]
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.repo = os.path.abspath(dvcs.path)
conf.matrix = {}
conf.build_cache_size = 0
repo = get_repo(conf)
def get_env():
return list(environment.get_environments(conf, None))[0]
env = get_env()
env.create()
# Check updating installed_commit_hash
assert env.installed_commit_hash == None
assert env._env_vars.get('ASV_COMMIT') == None
env.install_project(conf, repo, commit_hash)
assert env.installed_commit_hash == commit_hash
assert env._env_vars.get('ASV_COMMIT') == commit_hash
env = get_env()
assert env.installed_commit_hash == commit_hash
assert env._env_vars.get('ASV_COMMIT') == commit_hash
# Configuration change results to reinstall
env._project = "something"
assert env.installed_commit_hash == None
# Uninstall resets hash (but not ASV_COMMIT)
env = get_env()
env._uninstall_project()
assert env.installed_commit_hash == None
assert env._env_vars.get('ASV_COMMIT') != None
env = get_env()
assert env.installed_commit_hash == None
assert env._env_vars.get('ASV_COMMIT') == None
def _test_branches(conf, branch_commits):
r = repo.get_repo(conf)
branch_cache = BranchCache(conf, r)
assert len(conf.branches) == 2
commit_branches = {}
for branch in conf.branches:
commits = branch_cache.get_branch_commits(branch)
for commit in branch_commits[branch]:
assert commit in commits
commit_branches[commit] = branch
for commit, branch in commit_branches.items():
assert branch in branch_cache.get_branches(commit)
def _test_generic_repo(conf, tmpdir, hash_range, master, branch):
workcopy_dir = join(tmpdir, "workcopy")
r = repo.get_repo(conf)
r.checkout(workcopy_dir, master)
r.checkout(workcopy_dir, branch)
r.checkout(workcopy_dir, master)
hashes = r.get_hashes_from_range(hash_range)
assert len(hashes) == 4
dates = [r.get_date(hash) for hash in hashes]
assert dates == sorted(dates)[::-1]
tags = r.get_tags()
for tag in tags:
r.get_date_from_name(tag)
def generate_result_dir(tmpdir, dvcs, values, branches=None):
result_dir = join(tmpdir, "results")
os.makedirs(result_dir)
html_dir = join(tmpdir, "html")
machine_dir = join(result_dir, "tarzan")
os.makedirs(machine_dir)
if branches is None:
branches = [None]
conf = config.Config.from_json({
'results_dir': result_dir,
'html_dir': html_dir,
'repo': dvcs.path,
'project': 'asv',
'branches': branches or [None],
})
repo = get_repo(conf)
util.write_json(join(machine_dir, "machine.json"), {
'machine': 'tarzan',
'version': 1,
})
timestamp = datetime.datetime.utcnow()
params = None
for commit, value in values.items():
if isinstance(value, dict):
params = value["params"]
result = Results({"machine": "tarzan"}, {}, commit,
repo.get_date_from_name(commit), "2.7", None)
result.add_result("time_func", value, timestamp, timestamp)
result.save(result_dir)
util.write_json(join(result_dir, "benchmarks.json"), {
"time_func": {
"name": "time_func",
"params": params or [],
"param_names": params or [],
}
},
api_version=1)
return conf
def benchmarks_fixture(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
d['branches'] = ["master"]
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
return conf, repo, envs, commit_hash
def benchmarks_fixture(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
d['branches'] = ["master"]
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
return conf, repo, envs, commit_hash
def test_find_benchmarks_cwd_imports(tmpdir):
# Test that files in the directory above the benchmark suite are
# not importable
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
os.makedirs('benchmark')
with open(os.path.join('benchmark', '__init__.py'), 'w') as f:
pass
with open(os.path.join('benchmark', 'test.py'), 'w') as f:
f.write("""
try:
import this_should_really_not_be_here
raise AssertionError('This should not happen!')
except ImportError:
pass
def track_this():
return 0
""")
with open(os.path.join('this_should_really_not_be_here.py'), 'w') as f:
f.write("raise AssertionError('Should not be imported!')")
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [[0, 1]]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf,
repo,
envs, [commit_hash],
regex='track_this')
assert len(b) == 1
def generate_result_dir(tmpdir, dvcs, values, branches=None):
result_dir = join(tmpdir, "results")
os.makedirs(result_dir)
html_dir = join(tmpdir, "html")
machine_dir = join(result_dir, "tarzan")
os.makedirs(machine_dir)
if branches is None:
branches = [None]
conf = config.Config.from_json({
'results_dir': result_dir,
'html_dir': html_dir,
'repo': dvcs.path,
'project': 'asv',
'branches': branches or [None],
})
repo = get_repo(conf)
util.write_json(join(machine_dir, "machine.json"), {
'machine': 'tarzan',
})
timestamp = datetime.datetime.utcnow()
params = None
for commit, value in values.items():
if isinstance(value, dict):
params = value["params"]
result = Results({"machine": "tarzan"}, {}, commit,
repo.get_date_from_name(commit), "2.7", None)
result.add_result("time_func", value, timestamp, timestamp)
result.save(result_dir)
util.write_json(join(result_dir, "benchmarks.json"), {
"time_func": {
"name": "time_func",
"params": params or [],
"param_names": params or [],
}
}, api_version=1)
return conf
def test_repo(tmpdir):
conf = config.Config()
conf.project = six.text_type(tmpdir.join("repo"))
conf.repo = "https://github.com/spacetelescope/asv.git"
r = repo.get_repo(conf)
r.checkout("master")
r.checkout("gh-pages")
r.checkout("master")
hashes = r.get_hashes_from_range("ae0c27b65741..e6f382a704f7")
assert len(hashes) == 4
dates = [r.get_date(hash) for hash in hashes]
assert dates == sorted(dates)[::-1]
tags = r.get_tags()
for tag in tags:
r.get_date_from_tag(tag)
def __init__(self, conf: Config, python: str, requirements: dict) -> None:
"""
Parameters
----------
conf: Config instance
python : str
Version of Python. Must be of the form "MAJOR.MINOR".
requirements : dict
Dictionary mapping a PyPI package name to a version
identifier string.
"""
from nox.sessions import _normalize_path
# Need to checkout the project BEFORE the benchmark run - to access a noxfile.
self.project_temp_checkout = TemporaryDirectory(
prefix="nox_asv_checkout_")
repo = get_repo(conf)
repo.checkout(self.project_temp_checkout.name, conf.nox_setup_commit)
self.noxfile_rel_path = conf.noxfile_rel_path
self.setup_noxfile = (Path(self.project_temp_checkout.name) /
self.noxfile_rel_path)
self.nox_session_name = conf.nox_session_name
# Some duplication of parent code - need these attributes BEFORE
# running inherited code.
self._python = python
self._requirements = requirements
self._env_dir = conf.env_dir
# Prepare the actual environment path, to override self._path.
nox_envdir = str(Path(self._env_dir).absolute() / self.hashname)
nox_friendly_name = self._get_nox_session_name(python)
self._nox_path = Path(_normalize_path(nox_envdir, nox_friendly_name))
# For storing any extra conda requirements from asv.conf.json.
self._extra_reqs_path = self._nox_path / "asv-extra-reqs.yaml"
super().__init__(conf, python, requirements)
def test_find_benchmarks_cwd_imports(tmpdir):
# Test that files in the directory above the benchmark suite are
# not importable
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
os.makedirs('benchmark')
with open(os.path.join('benchmark', '__init__.py'), 'w') as f:
pass
with open(os.path.join('benchmark', 'test.py'), 'w') as f:
f.write("""
try:
import this_should_really_not_be_here
raise AssertionError('This should not happen!')
except ImportError:
pass
def track_this():
return 0
""")
with open(os.path.join('this_should_really_not_be_here.py'), 'w') as f:
f.write("raise AssertionError('Should not be imported!')")
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='track_this')
assert len(b) == 1
def _test_generic_repo(conf,
tmpdir,
hash_range="ae0c27b65741..e6f382a704f7",
master="master",
branch="gh-pages"):
workcopy_dir = six.text_type(tmpdir.join("workcopy"))
r = repo.get_repo(conf)
r.checkout(workcopy_dir, master)
r.checkout(workcopy_dir, branch)
r.checkout(workcopy_dir, master)
hashes = r.get_hashes_from_range(hash_range)
assert len(hashes) == 4
dates = [r.get_date(hash) for hash in hashes]
assert dates == sorted(dates)[::-1]
tags = r.get_tags()
for tag in tags:
r.get_date_from_name(tag)
def test_install_success(tmpdir):
# Check that install_project really installs the package. (gh-805)
# This may fail if pip in install_command e.g. gets confused by an .egg-info
# directory in its cwd to think the package is already installed.
tmpdir = six.text_type(tmpdir)
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
commit_hash = dvcs.get_branch_hashes()[0]
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.repo = os.path.abspath(dvcs.path)
conf.matrix = {}
conf.build_cache_size = 0
repo = get_repo(conf)
env = list(environment.get_environments(conf, None))[0]
env.create()
env.install_project(conf, repo, commit_hash)
env.run(['-c', 'import asv_test_repo as t, sys; sys.exit(0 if t.dummy_value == 0 else 1)'])
def test_import_failure_retry(tmpdir):
# Test that a different commit is tried on import failure
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
os.makedirs('benchmark')
with open(os.path.join('benchmark', '__init__.py'), 'w') as f:
f.write(
textwrap.dedent("""
import asv_test_repo
def time_foo():
pass
time_foo.number = asv_test_repo.dummy_value
if asv_test_repo.dummy_value == 0:
raise RuntimeError("fail discovery")
"""))
dvcs = tools.generate_test_repo(tmpdir, [2, 1, 0])
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = dvcs.path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hashes = dvcs.get_branch_hashes()
b = benchmarks.Benchmarks.discover(conf, repo, envs, commit_hashes)
assert len(b) == 1
assert b['time_foo']['number'] == 1
def test_import_failure_retry(tmpdir):
# Test that a different commit is tried on import failure
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
os.makedirs('benchmark')
with open(os.path.join('benchmark', '__init__.py'), 'w') as f:
f.write(textwrap.dedent("""
import asv_test_repo
def time_foo():
pass
time_foo.number = asv_test_repo.dummy_value
if asv_test_repo.dummy_value == 0:
raise RuntimeError("fail discovery")
"""))
dvcs = tools.generate_test_repo(tmpdir, [2, 1, 0])
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = dvcs.path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hashes = dvcs.get_branch_hashes()
b = benchmarks.Benchmarks.discover(conf, repo, envs, commit_hashes)
assert len(b) == 1
assert b['time_foo']['number'] == 1
def test_publish(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
result_dir = join(tmpdir, 'sample_results')
os.makedirs(result_dir)
os.makedirs(join(result_dir, 'cheetah'))
# Synthesize history with two branches that both have commits
result_files = [fn for fn in os.listdir(join(RESULT_DIR, 'cheetah'))
if fn.endswith('.json') and fn != 'machine.json']
result_files.sort()
master_values = list(range(len(result_files)*2//3))
branch_values = list(range(len(master_values), len(result_files)))
dvcs = tools.generate_test_repo(tmpdir, master_values, 'git',
[('master~6', 'some-branch', branch_values)])
# Copy and modify result files, fixing commit hashes and setting result
# dates to distinguish the two branches
master_commits = dvcs.get_branch_hashes('master')
only_branch = [x for x in dvcs.get_branch_hashes('some-branch')
if x not in master_commits]
commits = master_commits + only_branch
for k, item in enumerate(zip(result_files, commits)):
fn, commit = item
src = join(RESULT_DIR, 'cheetah', fn)
dst = join(result_dir, 'cheetah', commit[:8] + fn[8:])
data = util.load_json(src, cleanup=False)
data['commit_hash'] = commit
util.write_json(dst, data)
shutil.copyfile(join(RESULT_DIR, 'benchmarks.json'),
join(result_dir, 'benchmarks.json'))
shutil.copyfile(join(RESULT_DIR, 'cheetah', 'machine.json'),
join(result_dir, 'cheetah', 'machine.json'))
# Publish the synthesized data
conf = config.Config.from_json(
{'benchmark_dir': BENCHMARK_DIR,
'results_dir': result_dir,
'html_dir': join(tmpdir, 'html'),
'repo': dvcs.path,
'project': 'asv'})
tools.run_asv_with_conf(conf, 'publish')
# Check output
assert isfile(join(tmpdir, 'html', 'index.html'))
assert isfile(join(tmpdir, 'html', 'index.json'))
assert isfile(join(tmpdir, 'html', 'asv.js'))
assert isfile(join(tmpdir, 'html', 'asv.css'))
assert not isdir(join(tmpdir, 'html', 'graphs', 'Cython', 'arch-x86_64',
'branch-some-branch'))
assert not isdir(join(tmpdir, 'html', 'graphs', 'Cython-null', 'arch-x86_64',
'branch-some-branch'))
index = util.load_json(join(tmpdir, 'html', 'index.json'))
assert index['params']['branch'] == ['master']
repo = get_repo(conf)
revision_to_hash = dict((r, h) for h, r in six.iteritems(repo.get_revisions(commits)))
def check_file(branch, cython):
fn = join(tmpdir, 'html', 'graphs', cython, 'arch-x86_64', 'branch-' + branch,
'cpu-Intel(R) Core(TM) i5-2520M CPU @ 2.50GHz (4 cores)',
'machine-cheetah', 'numpy-1.8', 'os-Linux (Fedora 20)', 'python-2.7', 'ram-8.2G',
'time_coordinates.time_latitude.json')
data = util.load_json(fn, cleanup=False)
data_commits = [revision_to_hash[x[0]] for x in data]
if branch == "master":
assert all(c in master_commits for c in data_commits)
else:
# Must contains commits from some-branch
assert any(c in only_branch for c in data_commits)
# And commits from master
assert any(c in master_commits for c in data_commits)
# Check that revisions are strictly increasing
assert all(x[0] < y[0] for x, y in zip(data, data[1:]))
check_file("master", "Cython")
check_file("master", "Cython-null")
# Publish with branches set in the config
conf.branches = ['master', 'some-branch']
tools.run_asv_with_conf(conf, 'publish')
# Check output
check_file("master", "Cython")
check_file("master", "Cython-null")
check_file("some-branch", "Cython")
check_file("some-branch", "Cython-null")
index = util.load_json(join(tmpdir, 'html', 'index.json'))
assert index['params']['branch'] == ['master', 'some-branch']
assert index['params']['Cython'] == ['', None]
assert index['params']['ram'] == ['8.2G', 8804682956.8]
expected_graph_list = [{'Cython': cython, 'arch': 'x86_64',
'branch': branch,
'cpu': 'Intel(R) Core(TM) i5-2520M CPU @ 2.50GHz (4 cores)',
'machine': 'cheetah',
'numpy': '1.8',
'os': 'Linux (Fedora 20)',
'python': '2.7',
'ram': '8.2G'}
for cython in ["", None] for branch in ["master", "some-branch"]]
d = dict(expected_graph_list[0])
d['ram'] = 8804682956.8
expected_graph_list.append(d)
assert len(index['graph_param_list']) == len(expected_graph_list)
for item in expected_graph_list:
assert item in index['graph_param_list']
def test_custom_commands(tmpdir):
# check custom install/uninstall/build commands work
tmpdir = six.text_type(tmpdir)
dvcs = generate_test_repo(tmpdir, [0], dvcs_type='git')
build_py = os.path.abspath(os.path.join(tmpdir, 'build.py'))
install_py = os.path.abspath(os.path.join(tmpdir, 'install.py'))
uninstall_py = os.path.abspath(os.path.join(tmpdir, 'uninstall.py'))
conf = config.Config()
conf.env_dir = os.path.join(tmpdir, "env")
conf.pythons = [PYTHON_VER1]
conf.repo = os.path.abspath(dvcs.path)
conf.matrix = {}
conf.build_cache_size = 0
conf.build_command = ["python {0} {{build_cache_dir}}".format(quote(build_py))]
conf.install_command = ["python {0} {{env_dir}} {{build_cache_dir}}".format(quote(install_py))]
conf.uninstall_command = ["python {0} {{env_dir}}".format(quote(uninstall_py))]
with open(build_py, 'wb') as f:
f.write(b"import os, sys\n"
b"assert sys.argv[1] == os.environ['ASV_BUILD_CACHE_DIR']\n"
b"f = open(os.path.join(os.environ['ASV_BUILD_CACHE_DIR'], 'cached'), 'wb')\n"
b"f.write(b'data')\n"
b"f.close()\n")
with open(install_py, 'wb') as f:
f.write(b"import os, sys, shutil\n"
b"assert sys.argv[1] == os.environ['ASV_ENV_DIR']\n"
b"assert sys.argv[2] == os.environ['ASV_BUILD_CACHE_DIR']\n"
b"shutil.copyfile(os.path.join(os.environ['ASV_BUILD_CACHE_DIR'], 'cached'),\n"
b" os.path.join(os.environ['ASV_ENV_DIR'], 'installed'))\n")
with open(uninstall_py, 'wb') as f:
f.write(b"import os, sys\n"
b"assert sys.argv[1] == os.environ['ASV_ENV_DIR']\n"
b"fn = os.path.join(os.environ['ASV_ENV_DIR'], 'installed')\n"
b"if os.path.isfile(fn): os.unlink(fn)\n")
def get_env():
env = list(environment.get_environments(conf, None))[0]
env.create()
return env
env = get_env()
repo = get_repo(conf)
commit_hash = dvcs.get_branch_hashes()[0]
cache_dir = os.path.join(env._path, 'asv-build-cache')
cache_file = os.path.join(cache_dir, commit_hash, 'cached')
install_file = os.path.join(env._path, 'installed')
# Project installation should succeed with cache size 0,
# and not leave cache files around
env.install_project(conf, repo, commit_hash)
assert os.path.isfile(install_file)
assert not os.listdir(cache_dir)
env._set_installed_commit_hash(None)
# It should succed with nonzero cache size
conf.build_cache_size = 1
env = get_env()
env.install_project(conf, repo, commit_hash)
assert os.path.isfile(cache_file)
assert os.path.isfile(install_file)
# Explicitly check uninstall works
env._uninstall_project()
assert os.path.isfile(cache_file)
assert not os.path.isfile(install_file)
# Check reinstall uses cache and doesn't call build command
conf.build_command = ['python -c "import sys; sys.exit(1)"']
env = get_env()
env.install_project(conf, repo, commit_hash)
assert os.path.isfile(install_file)
assert os.path.isfile(cache_file)
# Bad install command should cause a failure
conf.install_command = ['python -c "import sys; sys.exit(1)"']
env = get_env()
with pytest.raises(util.ProcessError):
env.install_project(conf, repo, commit_hash)
def test_code_extraction(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex=r'^code_extraction\.')
expected_code = textwrap.dedent("""
def track_test():
# module-level 難
return 0
def setup():
# module-level
pass
def setup_cache():
# module-level
pass
""").strip()
bench = b['code_extraction.track_test']
assert bench['version'] == sha256(bench['code'].encode('utf-8')).hexdigest()
assert bench['code'] == expected_code
expected_code = textwrap.dedent("""
class MyClass:
def track_test(self):
# class-level 難
return 0
def setup():
# module-level
pass
class MyClass:
def setup(self):
# class-level
pass
def setup_cache(self):
# class-level
pass
""").strip()
bench = b['code_extraction.MyClass.track_test']
assert bench['version'] == sha256(bench['code'].encode('utf-8')).hexdigest()
if sys.version_info[:2] != (3, 2):
# Python 3.2 doesn't have __qualname__
assert bench['code'] == expected_code
def test_code_extraction(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf,
repo,
envs, [commit_hash],
regex=r'^code_extraction\.')
expected_code = textwrap.dedent("""
def track_test():
# module-level 難
return 0
def setup():
# module-level
pass
def setup_cache():
# module-level
pass
""").strip()
bench = b['code_extraction.track_test']
assert bench['version'] == sha256(
bench['code'].encode('utf-8')).hexdigest()
assert bench['code'] == expected_code
expected_code = textwrap.dedent("""
int track_pretty_source_test() {
return 0;
}
def setup():
# module-level
pass
def setup_cache():
# module-level
pass
""").strip()
bench = b['code_extraction.track_pretty_source_test']
assert bench['version'] == sha256(
bench['code'].encode('utf-8')).hexdigest()
assert bench['code'] == expected_code
expected_code = textwrap.dedent("""
class MyClass:
def track_test(self):
# class-level 難
return 0
def setup():
# module-level
pass
class MyClass:
def setup(self):
# class-level
pass
def setup_cache(self):
# class-level
pass
""").strip()
bench = b['code_extraction.MyClass.track_test']
assert bench['version'] == sha256(
bench['code'].encode('utf-8')).hexdigest()
if sys.version_info[:2] != (3, 2):
# Python 3.2 doesn't have __qualname__
assert bench['code'] == expected_code
def test_find_benchmarks(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
b = benchmarks.Benchmarks(conf, repo, envs, regex='secondary')
assert len(b) == 3
b = benchmarks.Benchmarks(conf, repo, envs, regex='example')
assert len(b) == 25
b = benchmarks.Benchmarks(conf,
repo,
envs,
regex='time_example_benchmark_1')
assert len(b) == 2
b = benchmarks.Benchmarks(conf,
repo,
envs,
regex=[
'time_example_benchmark_1',
'some regexp that does not match anything'
])
assert len(b) == 2
b = benchmarks.Benchmarks(conf, repo, envs, regex='custom')
assert sorted(b.keys()) == ['custom.time_function', 'custom.track_method']
b = benchmarks.Benchmarks(conf, repo, envs)
assert len(b) == 33
start_timestamp = datetime.datetime.utcnow()
b = benchmarks.Benchmarks(conf, repo, envs)
times = b.run_benchmarks(envs[0], profile=True, show_stderr=True)
end_timestamp = datetime.datetime.utcnow()
assert len(times) == len(b)
assert times['time_examples.TimeSuite.time_example_benchmark_1'][
'result'] != [None]
assert isinstance(
times['time_examples.TimeSuite.time_example_benchmark_1']['stats'][0]
['std'], float)
# The exact number of samples may vary if the calibration is not fully accurate
assert len(times['time_examples.TimeSuite.time_example_benchmark_1']
['samples'][0]) in (8, 9, 10)
# Benchmarks that raise exceptions should have a time of "None"
assert times['time_secondary.TimeSecondary.time_exception']['result'] == [
None
]
assert times['subdir.time_subdir.time_foo']['result'] != [None]
if not ON_PYPY:
# XXX: the memory benchmarks don't work on Pypy, since asizeof
# is CPython-only
assert times['mem_examples.mem_list']['result'][0] > 1000
assert times['time_secondary.track_value']['result'] == [42.0]
assert 'profile' in times['time_secondary.track_value']
assert 'stderr' in times['time_examples.time_with_warnings']
assert times['time_examples.time_with_warnings']['errcode'] != 0
assert times['time_examples.TimeWithBadTimer.time_it']['result'] == [0.0]
assert times['params_examples.track_param']['params'] == [[
"<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"
]]
assert times['params_examples.track_param']['result'] == [42, 42]
assert times['params_examples.mem_param']['params'] == [['10', '20'],
['2', '3']]
assert len(times['params_examples.mem_param']['result']) == 2 * 2
assert times['params_examples.ParamSuite.track_value']['params'] == [[
"'a'", "'b'", "'c'"
]]
assert times['params_examples.ParamSuite.track_value']['result'] == [
1 + 0, 2 + 0, 3 + 0
]
assert isinstance(times['params_examples.TuningTest.time_it']['result'][0],
float)
assert isinstance(times['params_examples.time_skip']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result'][1], float)
assert util.is_nan(times['params_examples.time_skip']['result'][2])
assert times['peakmem_examples.peakmem_list']['result'][0] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example']['result'] == [
500
]
assert times['cache_examples.ClassLevelSetup.track_example2'][
'result'] == [500]
assert times['cache_examples.track_cache_foo']['result'] == [42]
assert times['cache_examples.track_cache_bar']['result'] == [12]
assert times['cache_examples.track_my_cache_foo']['result'] == [0]
assert times['cache_examples.ClassLevelSetupFail.track_fail'][
'result'] == None
assert 'raise RuntimeError()' in times[
'cache_examples.ClassLevelSetupFail.track_fail']['stderr']
assert times['cache_examples.ClassLevelCacheTimeout.track_fail'][
'result'] == None
assert times['cache_examples.ClassLevelCacheTimeoutSuccess.track_success'][
'result'] == [0]
profile_path = join(tmpdir, 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value']['profile'])
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split(
) == expected
# Calibration of iterations should not rerun setup
expected = (['setup'] * 2, ['setup'] * 3)
assert times['time_examples.TimeWithRepeatCalibrate.time_it'][
'stderr'].split() in expected
# Check run time timestamps
for name, result in times.items():
assert result['started_at'] >= start_timestamp
assert result['ended_at'] >= result['started_at']
assert result['ended_at'] <= end_timestamp
def test_find_benchmarks(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
shutil.copytree(BENCHMARK_DIR, 'benchmark')
d = {}
d.update(ASV_CONF_JSON)
d['env_dir'] = "env"
d['benchmark_dir'] = 'benchmark'
d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
conf = config.Config.from_json(d)
repo = get_repo(conf)
envs = list(environment.get_environments(conf, None))
commit_hash = repo.get_hash_from_name(repo.get_branch_name())
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='secondary')
assert len(b) == 3
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='example')
assert len(b) == 25
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex='time_example_benchmark_1')
assert len(b) == 2
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash],
regex=['time_example_benchmark_1',
'some regexp that does not match anything'])
assert len(b) == 2
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash], regex='custom')
assert sorted(b.keys()) == ['custom.time_function', 'custom.track_method',
'named.track_custom_pretty_name']
assert 'pretty_name' not in b['custom.track_method']
assert b['custom.time_function']['pretty_name'] == 'My Custom Function'
assert b['named.track_custom_pretty_name']['pretty_name'] == 'this.is/the.answer'
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
assert len(b) == 35
assert 'named.OtherSuite.track_some_func' in b
start_timestamp = datetime.datetime.utcnow()
b = benchmarks.Benchmarks.discover(conf, repo, envs, [commit_hash])
times = b.run_benchmarks(envs[0], profile=True, show_stderr=True)
end_timestamp = datetime.datetime.utcnow()
assert len(times) == len(b)
assert times[
'time_examples.TimeSuite.time_example_benchmark_1']['result'] != [None]
assert isinstance(times['time_examples.TimeSuite.time_example_benchmark_1']['stats'][0]['std'], float)
# The exact number of samples may vary if the calibration is not fully accurate
assert len(times['time_examples.TimeSuite.time_example_benchmark_1']['samples'][0]) >= 5
# Benchmarks that raise exceptions should have a time of "None"
assert times[
'time_secondary.TimeSecondary.time_exception']['result'] == [None]
assert times[
'subdir.time_subdir.time_foo']['result'] != [None]
if not ON_PYPY:
# XXX: the memory benchmarks don't work on Pypy, since asizeof
# is CPython-only
assert times[
'mem_examples.mem_list']['result'][0] > 1000
assert times[
'time_secondary.track_value']['result'] == [42.0]
assert 'profile' in times[
'time_secondary.track_value']
assert 'stderr' in times[
'time_examples.time_with_warnings']
assert times['time_examples.time_with_warnings']['errcode'] != 0
assert times['time_examples.TimeWithBadTimer.time_it']['result'] == [0.0]
assert times['params_examples.track_param']['params'] == [["<class 'benchmark.params_examples.ClassOne'>",
"<class 'benchmark.params_examples.ClassTwo'>"]]
assert times['params_examples.track_param']['result'] == [42, 42]
assert times['params_examples.mem_param']['params'] == [['10', '20'], ['2', '3']]
assert len(times['params_examples.mem_param']['result']) == 2*2
assert times['params_examples.ParamSuite.track_value']['params'] == [["'a'", "'b'", "'c'"]]
assert times['params_examples.ParamSuite.track_value']['result'] == [1+0, 2+0, 3+0]
assert isinstance(times['params_examples.TuningTest.time_it']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result'][0], float)
assert isinstance(times['params_examples.time_skip']['result'][1], float)
assert util.is_nan(times['params_examples.time_skip']['result'][2])
assert times['peakmem_examples.peakmem_list']['result'][0] >= 4 * 2**20
assert times['cache_examples.ClassLevelSetup.track_example']['result'] == [500]
assert times['cache_examples.ClassLevelSetup.track_example2']['result'] == [500]
assert times['cache_examples.track_cache_foo']['result'] == [42]
assert times['cache_examples.track_cache_bar']['result'] == [12]
assert times['cache_examples.track_my_cache_foo']['result'] == [0]
assert times['cache_examples.ClassLevelSetupFail.track_fail']['result'] == None
assert 'raise RuntimeError()' in times['cache_examples.ClassLevelSetupFail.track_fail']['stderr']
assert times['cache_examples.ClassLevelCacheTimeout.track_fail']['result'] == None
assert times['cache_examples.ClassLevelCacheTimeoutSuccess.track_success']['result'] == [0]
profile_path = join(tmpdir, 'test.profile')
with open(profile_path, 'wb') as fd:
fd.write(times['time_secondary.track_value']['profile'])
pstats.Stats(profile_path)
# Check for running setup on each repeat (one extra run from profile)
# The output would contain error messages if the asserts in the benchmark fail.
expected = ["<%d>" % j for j in range(1, 12)]
assert times['time_examples.TimeWithRepeat.time_it']['stderr'].split() == expected
# Calibration of iterations should not rerun setup
expected = (['setup']*2, ['setup']*3)
assert times['time_examples.TimeWithRepeatCalibrate.time_it']['stderr'].split() in expected
# Check run time timestamps
for name, result in times.items():
assert result['started_at'] >= start_timestamp
assert result['ended_at'] >= result['started_at']
assert result['ended_at'] <= end_timestamp
