def test_run_publish(capfd, basic_conf):
tmpdir, local, conf, machine_file = basic_conf
# Tests a typical complete run/publish workflow
Run.run(conf, range_spec="master~5..master", steps=2,
_machine_file=machine_file, quick=True, show_stderr=True)
text, err = capfd.readouterr()
assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5
assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2
assert 'asv: benchmark timed out (timeout 0.1s)' in text
Publish.run(conf)
assert isfile(join(tmpdir, 'html', 'index.html'))
assert isfile(join(tmpdir, 'html', 'index.json'))
assert isfile(join(tmpdir, 'html', 'asv.js'))
assert isfile(join(tmpdir, 'html', 'asv.css'))
# Check parameterized test json data format
filename = glob.glob(join(tmpdir, 'html', 'graphs', 'arch-x86_64', 'branch-master',
'colorama-0.3.3', 'cpu-Blazingly fast', 'machine-orangutan',
'os-GNU', 'Linux', 'python-*', 'ram-128GB',
'six', 'params_examples.time_skip.json'))[0]
with open(filename, 'r') as fp:
data = json.load(fp)
assert len(data) == 2
assert isinstance(data[0][0], six.integer_types) # date
assert len(data[0][1]) == 3
assert len(data[1][1]) == 3
assert isinstance(data[0][1][0], float)
assert isinstance(data[0][1][1], float)
assert data[0][1][2] is None
# Check that the skip options work
capfd.readouterr()
Run.run(conf, range_spec="master~5..master", steps=2,
_machine_file=join(tmpdir, 'asv-machine.json'), quick=True,
skip_successful=True, skip_failed=True)
Run.run(conf, range_spec="master~5..master", steps=2,
_machine_file=join(tmpdir, 'asv-machine.json'), quick=True,
skip_existing_commits=True)
text, err = capfd.readouterr()
assert 'Running benchmarks.' not in text
# Check EXISTING works
Run.run(conf, range_spec="EXISTING",
_machine_file=machine_file, quick=True)
# Remove the benchmarks.json file to make sure publish can
# regenerate it
os.remove(join(tmpdir, "results_workflow", "benchmarks.json"))
Publish.run(conf)
def test_publish(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
conf = config.Config.from_json(
{'results_dir': RESULT_DIR,
'html_dir': join(tmpdir, 'html'),
'repo': 'https://github.com/spacetelescope/asv.git',
'project': 'asv'})
Publish.run(conf)
assert exists(join(tmpdir, 'html', 'index.html'))
assert exists(join(tmpdir, 'html', 'index.json'))
assert exists(join(tmpdir, 'html', 'asv.js'))
assert exists(join(tmpdir, 'html', 'asv.css'))
def test_publish(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
conf = config.Config.from_json({
'results_dir': RESULT_DIR,
'html_dir': join(tmpdir, 'html'),
'repo': 'https://github.com/spacetelescope/asv.git',
'project': 'asv'
})
Publish.run(conf)
assert exists(join(tmpdir, 'html', 'index.html'))
assert exists(join(tmpdir, 'html', 'index.json'))
assert exists(join(tmpdir, 'html', 'asv.js'))
assert exists(join(tmpdir, 'html', 'asv.css'))
def test_workflow(tmpdir):
# Tests a typical complete run/publish workflow
tmpdir = six.text_type(tmpdir)
local = abspath(dirname(__file__))
os.chdir(tmpdir)
shutil.copyfile(join(local, 'asv-machine.json'),
join(tmpdir, 'asv-machine.json'))
conf = config.Config.from_json({
'env_dir': join(tmpdir, 'env'),
'benchmark_dir': join(local, 'benchmark'),
'results_dir': join(tmpdir, 'results_workflow'),
'html_dir': join(tmpdir, 'html'),
'repo': 'https://github.com/spacetelescope/asv.git',
'project': 'asv',
'matrix': {
"six": [None],
"psutil": ["1.2", "1.1"]
}
})
Run.run(conf, range_spec="initial..master", steps=2,
_machine_file=join(tmpdir, 'asv-machine.json'), quick=True)
assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5
assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2
Publish.run(conf)
assert exists(join(tmpdir, 'html', 'index.html'))
assert exists(join(tmpdir, 'html', 'index.json'))
assert exists(join(tmpdir, 'html', 'asv.js'))
assert exists(join(tmpdir, 'html', 'asv.css'))
Run.run(conf, range_spec="EXISTING",
_machine_file=join(tmpdir, 'asv-machine.json'), quick=True)
# Remove the benchmarks.json file to make sure publish can
# regenerate it
os.remove(join(tmpdir, "results_workflow", "benchmarks.json"))
Publish.run(conf)
def test_workflow(tmpdir):
# Tests a typical complete run/publish workflow
tmpdir = six.text_type(tmpdir)
local = abspath(dirname(__file__))
os.chdir(tmpdir)
conf = config.Config.from_json({
'env_dir': join(tmpdir, 'env'),
'benchmark_dir': join(local, 'benchmark'),
'results_dir': join(tmpdir, 'results_workflow'),
'html_dir': join(tmpdir, 'html'),
'repo': 'https://github.com/spacetelescope/asv.git',
'project': 'asv',
'matrix': {
"six": [None],
"psutil": ["1.2", "1.1"]
}
})
Run.run(conf, range_spec="initial..master", steps=2,
_machine_file=join(local, 'asv-machine.json'), quick=True)
assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5
assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2
Publish.run(conf)
assert exists(join(tmpdir, 'html', 'index.html'))
assert exists(join(tmpdir, 'html', 'index.json'))
assert exists(join(tmpdir, 'html', 'asv.js'))
assert exists(join(tmpdir, 'html', 'asv.css'))
Run.run(conf, range_spec="EXISTING",
_machine_file=join(local, 'asv-machine.json'), quick=True)
# Remove the benchmarks.json file to make sure publish can
# regenerate it
os.remove(join(tmpdir, "results_workflow", "benchmarks.json"))
Publish.run(conf)
def basic_html(request):
tmpdir = tempfile.mkdtemp()
request.addfinalizer(lambda: shutil.rmtree(tmpdir))
local = abspath(dirname(__file__))
cwd = os.getcwd()
os.chdir(tmpdir)
try:
machine_file = join(tmpdir, 'asv-machine.json')
shutil.copyfile(join(local, 'asv-machine.json'),
machine_file)
dvcs = tools.generate_test_repo(tmpdir, list(range(10)))
repo_path = dvcs.path
conf = config.Config.from_json({
'env_dir': join(tmpdir, 'env'),
'benchmark_dir': join(local, 'benchmark'),
'results_dir': join(tmpdir, 'results_workflow'),
'html_dir': join(tmpdir, 'html'),
'repo': repo_path,
'dvcs': 'git',
'project': 'asv',
'matrix': {
"six": [None],
"colorama": ["0.3.1", "0.3.3"]
}
})
Run.run(conf, range_spec="master~5..master", steps=3,
_machine_file=machine_file, quick=True)
Publish.run(conf)
finally:
os.chdir(cwd)
return conf, dvcs
def test_web_regressions(browser, tmpdir):
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains
tmpdir = six.text_type(tmpdir)
local = abspath(dirname(__file__))
cwd = os.getcwd()
os.chdir(tmpdir)
try:
machine_file = join(tmpdir, 'asv-machine.json')
shutil.copyfile(join(local, 'asv-machine.json'),
machine_file)
values = [[x]*2 for x in [0, 0, 0, 0, 0,
1, 1, 1, 1, 1,
3, 3, 3, 3, 3,
2, 2, 2, 2, 2]]
dvcs = tools.generate_test_repo(tmpdir, values)
repo_path = dvcs.path
first_tested_commit_hash = dvcs.get_hash('master~14')
conf = config.Config.from_json({
'env_dir': join(tmpdir, 'env'),
'benchmark_dir': join(local, 'benchmark'),
'results_dir': join(tmpdir, 'results_workflow'),
'html_dir': join(tmpdir, 'html'),
'repo': repo_path,
'dvcs': 'git',
'project': 'asv',
'matrix': {},
'regressions_first_commits': {
'.*': first_tested_commit_hash
},
})
Run.run(conf, range_spec="ALL", bench='params_examples.track_find_test',
_machine_file=machine_file, show_stderr=True, quick=True)
Publish.run(conf)
finally:
os.chdir(cwd)
bad_commit_hash = dvcs.get_hash('master~9')
with tools.preview(conf.html_dir) as base_url:
browser.get(base_url)
regressions_btn = browser.find_element_by_link_text('Show regressions')
regressions_btn.click()
# Check that the expected links appear in the table
regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)')
regression_2 = browser.find_element_by_link_text('params_examples.track_find_test(2)')
bad_hash_link = browser.find_element_by_link_text(bad_commit_hash[:8])
href = regression_1.get_attribute('href')
assert '/#params_examples.track_find_test?' in href
assert 'time=' in href
# Sort the tables vs. benchmark name (PhantomJS doesn't allow doing it via actionchains)
browser.execute_script("$('thead th').eq(0).stupidsort('asc')")
WebDriverWait(browser, 5).until(EC.text_to_be_present_in_element(
('xpath', '//table[1]/tbody/tr[1]/td[1]'), 'params_examples.track_find_test(1)'
))
# Check the contents of the table
table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr')
assert len(table_rows) == 2
cols1 = [td.text for td in table_rows[0].find_elements_by_xpath('td')]
cols2 = [td.text for td in table_rows[1].find_elements_by_xpath('td')]
assert cols1[0] == 'params_examples.track_find_test(1)'
assert cols2[0] == 'params_examples.track_find_test(2)'
assert re.match(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d+Z$', cols1[1])
assert re.match(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d+Z$', cols2[1])
assert cols1[2:] == [bad_commit_hash[:8], '2.00x', '1.00', '2.00', 'Ignore']
assert cols2[2:] == [bad_commit_hash[:8], '2.00x', '1.00', '2.00', 'Ignore']
# Check that the ignore buttons work as expected
buttons = [button for button in browser.find_elements_by_xpath('//button')
if button.text == 'Ignore']
buttons[0].click()
# The button should disappear, together with the link
WebDriverWait(browser, 5).until_not(EC.visibility_of(buttons[0]))
WebDriverWait(browser, 5).until_not(EC.visibility_of(regression_1))
table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr')
assert len(table_rows) == 1
# There's a second button for showing the links, clicking
# which makes the elements reappear
show_button = [button for button in browser.find_elements_by_xpath('//button')
if button.text == 'Show ignored regressions...'][0]
show_button.click()
regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)')
WebDriverWait(browser, 5).until(EC.visibility_of(regression_1))
table_rows = browser.find_elements_by_xpath('//table[2]/tbody/tr')
assert len(table_rows) == 1
# There's a config sample element
pre_div = browser.find_element_by_xpath('//pre')
assert "params_examples\\\\.track_find_test\\\\(1\\\\)" in pre_div.text
# There's an unignore button that moves the element back to the main table
unignore_button = [button for button in browser.find_elements_by_xpath('//button')
if button.text == 'Unignore'][0]
unignore_button.click()
browser.find_elements_by_xpath('//table[1]/tbody/tr[2]') # wait until the table has two rows
table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr')
assert len(table_rows) == 2
# Check that a plot of some sort appears on mouseover. The
# page needs to be scrolled first so that the mouseover popup
# has enough space to appear.
regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)')
y = regression_1.location['y']
browser.execute_script('window.scrollTo(0, {0})'.format(y - 200))
chain = ActionChains(browser)
chain.move_to_element(regression_1)
chain.perform()
popover = browser.find_element_by_css_selector('div.popover-content')
flotplot = browser.find_element_by_css_selector('canvas.flot-base')
def test_publish(tmpdir):
tmpdir = six.text_type(tmpdir)
os.chdir(tmpdir)
result_dir = join(tmpdir, 'sample_results')
os.makedirs(result_dir)
os.makedirs(join(result_dir, 'cheetah'))
# Synthesize history with two branches that both have commits
result_files = [fn for fn in os.listdir(join(RESULT_DIR, 'cheetah'))
if fn.endswith('.json') and fn != 'machine.json']
master_values = list(range(len(result_files)*2//3))
branch_values = list(range(len(master_values), len(result_files)))
dvcs = tools.generate_test_repo(tmpdir, master_values, 'git',
[('master~6', 'some-branch', branch_values)])
# Copy and modify result files, fixing commit hashes and setting result
# dates to distinguish the two branches
master_commits = dvcs.get_branch_hashes('master')
only_branch = [x for x in dvcs.get_branch_hashes('some-branch')
if x not in master_commits]
commits = master_commits + only_branch
for k, item in enumerate(zip(result_files, commits)):
fn, commit = item
src = join(RESULT_DIR, 'cheetah', fn)
dst = join(result_dir, 'cheetah', commit[:8] + fn[8:])
data = util.load_json(src, cleanup=False)
data['commit_hash'] = commit
if commit in only_branch:
data['date'] = -k
else:
data['date'] = k
util.write_json(dst, data)
shutil.copyfile(join(RESULT_DIR, 'benchmarks.json'),
join(result_dir, 'benchmarks.json'))
shutil.copyfile(join(RESULT_DIR, 'cheetah', 'machine.json'),
join(result_dir, 'cheetah', 'machine.json'))
# Publish the synthesized data
conf = config.Config.from_json(
{'benchmark_dir': BENCHMARK_DIR,
'results_dir': result_dir,
'html_dir': join(tmpdir, 'html'),
'repo': dvcs.path,
'project': 'asv'})
Publish.run(conf)
# Check output
assert isfile(join(tmpdir, 'html', 'index.html'))
assert isfile(join(tmpdir, 'html', 'index.json'))
assert isfile(join(tmpdir, 'html', 'asv.js'))
assert isfile(join(tmpdir, 'html', 'asv.css'))
assert not isdir(join(tmpdir, 'html', 'graphs', 'Cython', 'arch-x86_64',
'branch-some-branch'))
index = util.load_json(join(tmpdir, 'html', 'index.json'))
assert index['params']['branch'] == ['master']
def check_file(branch):
fn = join(tmpdir, 'html', 'graphs', 'Cython', 'arch-x86_64', 'branch-' + branch,
'cpu-Intel(R) Core(TM) i5-2520M CPU @ 2.50GHz (4 cores)',
'machine-cheetah', 'numpy-1.8', 'os-Linux (Fedora 20)', 'python-2.7', 'ram-8.2G',
'time_coordinates.time_latitude.json')
data = util.load_json(fn, cleanup=False)
if branch == 'master':
# we set all dates positive for master above
assert all(x[0] >= 0 for x in data)
else:
# we set some dates negative for some-branch above
assert any(x[0] < 0 for x in data) and any(x[0] >= 0 for x in data)
check_file("master")
# Publish with branches set in the config
conf.branches = ['master', 'some-branch']
Publish.run(conf)
# Check output
check_file("master")
check_file("some-branch")
index = util.load_json(join(tmpdir, 'html', 'index.json'))
assert index['params']['branch'] == ['master', 'some-branch']
def create_benchmark_dataframe(group_by="name", use_branch_names=False):
# if we are in an asv subprocess, use ASV_CONF_DIR to load the config
repo_dirname = os.environ.get("ASV_CONF_DIR", _find_asv_root())
config_path = os.path.join(repo_dirname, "asv.conf.json")
config = Config.load(config_path)
# results_dir is a relative path to the benchmarks repository. If the
# directory where the code is run is not the benchmark repository, then
# loading the results will fail.
config.results_dir = os.path.join(repo_dirname, "results")
benchmarks = Benchmarks.load(config)
results = defaultdict(dict)
metadata_levels = [
"type",
"name",
"class",
"file",
"version",
"commit_hash",
"date",
]
if isinstance(group_by, str):
group_by = [group_by]
levels_to_group_by = group_by
levels_to_concat_on = [
l for l in metadata_levels if l not in levels_to_group_by
]
commit_to_branch_map = _get_commit_to_branch_map(config.repo)
for single_env_result in Publish.iter_results(config, benchmarks):
benchmark_metadata = {
"version": single_env_result._params["python"],
"commit_hash": single_env_result._commit_hash,
"date": single_env_result._date,
}
if use_branch_names:
benchmark_metadata["commit_hash"] = commit_to_branch_map.get(
benchmark_metadata["commit_hash"],
benchmark_metadata["commit_hash"])
for b_name, params in single_env_result._benchmark_params.items():
unquoted_params = _remove_quotes(params)
filename, classname, benchname = b_name.split(".")
_benchmark = benchmarks[b_name]
b_type, param_names = _benchmark["type"], _benchmark["param_names"]
benchmark_metadata.update({
"type": b_type,
"file": filename,
"class": classname,
"name": benchname,
})
values_to_group_by = tuple(
[benchmark_metadata[key] for key in levels_to_group_by])
values_to_concat_on = tuple(
[benchmark_metadata[key] for key in levels_to_concat_on])
# this is dangerous because we there is no reason the results
# order follow the carthesian product of the parameter space,
# however empirically it seems to be the case
params_with_infered_types = []
_results = single_env_result._results[b_name]
for params in unquoted_params:
params_with_infered_types.append(
pd.to_numeric(params, errors="ignore"))
if params_with_infered_types != []:
mi = pd.MultiIndex.from_product(params_with_infered_types,
names=param_names)
else:
# benchmark is not parametrized, make index a simple range
# index
mi = pd.RangeIndex(len(_results))
if len(_results) != len(mi):
# if a benchmark fails, single_env_result._results[b_name]
# only consists of [None]
assert _results == [None], 'unexpected benchmark result'
continue
_results = pd.Series(_results, index=mi)
_results.dropna(inplace=True)
results[values_to_group_by][values_to_concat_on] = _results
clean_result = {}
for k, v in results.items():
if len(k) == 1:
# if key if a list of length one, convert it to a string by taking
# its only element
clean_result[k[0]] = pd.concat(v, names=levels_to_concat_on)
elif len(k) == 0:
# if key is of length 0, there is only one element, so, return the
# underlying dict
clean_result = pd.concat(v, names=levels_to_concat_on)
else:
clean_result[k] = pd.concat(v, names=levels_to_concat_on)
return clean_result
