def test_run_publish(capfd, basic_conf): tmpdir, local, conf, machine_file = basic_conf # Tests a typical complete run/publish workflow Run.run(conf, range_spec="master~5..master", steps=2, _machine_file=machine_file, quick=True, show_stderr=True) text, err = capfd.readouterr() assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5 assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2 assert 'asv: benchmark timed out (timeout 0.1s)' in text Publish.run(conf) assert isfile(join(tmpdir, 'html', 'index.html')) assert isfile(join(tmpdir, 'html', 'index.json')) assert isfile(join(tmpdir, 'html', 'asv.js')) assert isfile(join(tmpdir, 'html', 'asv.css')) # Check parameterized test json data format filename = glob.glob(join(tmpdir, 'html', 'graphs', 'arch-x86_64', 'branch-master', 'colorama-0.3.3', 'cpu-Blazingly fast', 'machine-orangutan', 'os-GNU', 'Linux', 'python-*', 'ram-128GB', 'six', 'params_examples.time_skip.json'))[0] with open(filename, 'r') as fp: data = json.load(fp) assert len(data) == 2 assert isinstance(data[0][0], six.integer_types) # date assert len(data[0][1]) == 3 assert len(data[1][1]) == 3 assert isinstance(data[0][1][0], float) assert isinstance(data[0][1][1], float) assert data[0][1][2] is None # Check that the skip options work capfd.readouterr() Run.run(conf, range_spec="master~5..master", steps=2, _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, skip_successful=True, skip_failed=True) Run.run(conf, range_spec="master~5..master", steps=2, _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, skip_existing_commits=True) text, err = capfd.readouterr() assert 'Running benchmarks.' not in text # Check EXISTING works Run.run(conf, range_spec="EXISTING", _machine_file=machine_file, quick=True) # Remove the benchmarks.json file to make sure publish can # regenerate it os.remove(join(tmpdir, "results_workflow", "benchmarks.json")) Publish.run(conf)
def test_publish(tmpdir): tmpdir = six.text_type(tmpdir) os.chdir(tmpdir) conf = config.Config.from_json( {'results_dir': RESULT_DIR, 'html_dir': join(tmpdir, 'html'), 'repo': 'https://github.com/spacetelescope/asv.git', 'project': 'asv'}) Publish.run(conf) assert exists(join(tmpdir, 'html', 'index.html')) assert exists(join(tmpdir, 'html', 'index.json')) assert exists(join(tmpdir, 'html', 'asv.js')) assert exists(join(tmpdir, 'html', 'asv.css'))
def test_publish(tmpdir): tmpdir = six.text_type(tmpdir) os.chdir(tmpdir) conf = config.Config.from_json({ 'results_dir': RESULT_DIR, 'html_dir': join(tmpdir, 'html'), 'repo': 'https://github.com/spacetelescope/asv.git', 'project': 'asv' }) Publish.run(conf) assert exists(join(tmpdir, 'html', 'index.html')) assert exists(join(tmpdir, 'html', 'index.json')) assert exists(join(tmpdir, 'html', 'asv.js')) assert exists(join(tmpdir, 'html', 'asv.css'))
def test_workflow(tmpdir): # Tests a typical complete run/publish workflow tmpdir = six.text_type(tmpdir) local = abspath(dirname(__file__)) os.chdir(tmpdir) shutil.copyfile(join(local, 'asv-machine.json'), join(tmpdir, 'asv-machine.json')) conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': 'https://github.com/spacetelescope/asv.git', 'project': 'asv', 'matrix': { "six": [None], "psutil": ["1.2", "1.1"] } }) Run.run(conf, range_spec="initial..master", steps=2, _machine_file=join(tmpdir, 'asv-machine.json'), quick=True) assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5 assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2 Publish.run(conf) assert exists(join(tmpdir, 'html', 'index.html')) assert exists(join(tmpdir, 'html', 'index.json')) assert exists(join(tmpdir, 'html', 'asv.js')) assert exists(join(tmpdir, 'html', 'asv.css')) Run.run(conf, range_spec="EXISTING", _machine_file=join(tmpdir, 'asv-machine.json'), quick=True) # Remove the benchmarks.json file to make sure publish can # regenerate it os.remove(join(tmpdir, "results_workflow", "benchmarks.json")) Publish.run(conf)
def test_workflow(tmpdir): # Tests a typical complete run/publish workflow tmpdir = six.text_type(tmpdir) local = abspath(dirname(__file__)) os.chdir(tmpdir) conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': 'https://github.com/spacetelescope/asv.git', 'project': 'asv', 'matrix': { "six": [None], "psutil": ["1.2", "1.1"] } }) Run.run(conf, range_spec="initial..master", steps=2, _machine_file=join(local, 'asv-machine.json'), quick=True) assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5 assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2 Publish.run(conf) assert exists(join(tmpdir, 'html', 'index.html')) assert exists(join(tmpdir, 'html', 'index.json')) assert exists(join(tmpdir, 'html', 'asv.js')) assert exists(join(tmpdir, 'html', 'asv.css')) Run.run(conf, range_spec="EXISTING", _machine_file=join(local, 'asv-machine.json'), quick=True) # Remove the benchmarks.json file to make sure publish can # regenerate it os.remove(join(tmpdir, "results_workflow", "benchmarks.json")) Publish.run(conf)
def basic_html(request): tmpdir = tempfile.mkdtemp() request.addfinalizer(lambda: shutil.rmtree(tmpdir)) local = abspath(dirname(__file__)) cwd = os.getcwd() os.chdir(tmpdir) try: machine_file = join(tmpdir, 'asv-machine.json') shutil.copyfile(join(local, 'asv-machine.json'), machine_file) dvcs = tools.generate_test_repo(tmpdir, list(range(10))) repo_path = dvcs.path conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': repo_path, 'dvcs': 'git', 'project': 'asv', 'matrix': { "six": [None], "colorama": ["0.3.1", "0.3.3"] } }) Run.run(conf, range_spec="master~5..master", steps=3, _machine_file=machine_file, quick=True) Publish.run(conf) finally: os.chdir(cwd) return conf, dvcs
def test_web_regressions(browser, tmpdir): from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver import ActionChains tmpdir = six.text_type(tmpdir) local = abspath(dirname(__file__)) cwd = os.getcwd() os.chdir(tmpdir) try: machine_file = join(tmpdir, 'asv-machine.json') shutil.copyfile(join(local, 'asv-machine.json'), machine_file) values = [[x]*2 for x in [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2]] dvcs = tools.generate_test_repo(tmpdir, values) repo_path = dvcs.path first_tested_commit_hash = dvcs.get_hash('master~14') conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': repo_path, 'dvcs': 'git', 'project': 'asv', 'matrix': {}, 'regressions_first_commits': { '.*': first_tested_commit_hash }, }) Run.run(conf, range_spec="ALL", bench='params_examples.track_find_test', _machine_file=machine_file, show_stderr=True, quick=True) Publish.run(conf) finally: os.chdir(cwd) bad_commit_hash = dvcs.get_hash('master~9') with tools.preview(conf.html_dir) as base_url: browser.get(base_url) regressions_btn = browser.find_element_by_link_text('Show regressions') regressions_btn.click() # Check that the expected links appear in the table regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)') regression_2 = browser.find_element_by_link_text('params_examples.track_find_test(2)') bad_hash_link = browser.find_element_by_link_text(bad_commit_hash[:8]) href = regression_1.get_attribute('href') assert '/#params_examples.track_find_test?' in href assert 'time=' in href # Sort the tables vs. benchmark name (PhantomJS doesn't allow doing it via actionchains) browser.execute_script("$('thead th').eq(0).stupidsort('asc')") WebDriverWait(browser, 5).until(EC.text_to_be_present_in_element( ('xpath', '//table[1]/tbody/tr[1]/td[1]'), 'params_examples.track_find_test(1)' )) # Check the contents of the table table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr') assert len(table_rows) == 2 cols1 = [td.text for td in table_rows[0].find_elements_by_xpath('td')] cols2 = [td.text for td in table_rows[1].find_elements_by_xpath('td')] assert cols1[0] == 'params_examples.track_find_test(1)' assert cols2[0] == 'params_examples.track_find_test(2)' assert re.match(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d+Z$', cols1[1]) assert re.match(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d+Z$', cols2[1]) assert cols1[2:] == [bad_commit_hash[:8], '2.00x', '1.00', '2.00', 'Ignore'] assert cols2[2:] == [bad_commit_hash[:8], '2.00x', '1.00', '2.00', 'Ignore'] # Check that the ignore buttons work as expected buttons = [button for button in browser.find_elements_by_xpath('//button') if button.text == 'Ignore'] buttons[0].click() # The button should disappear, together with the link WebDriverWait(browser, 5).until_not(EC.visibility_of(buttons[0])) WebDriverWait(browser, 5).until_not(EC.visibility_of(regression_1)) table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr') assert len(table_rows) == 1 # There's a second button for showing the links, clicking # which makes the elements reappear show_button = [button for button in browser.find_elements_by_xpath('//button') if button.text == 'Show ignored regressions...'][0] show_button.click() regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)') WebDriverWait(browser, 5).until(EC.visibility_of(regression_1)) table_rows = browser.find_elements_by_xpath('//table[2]/tbody/tr') assert len(table_rows) == 1 # There's a config sample element pre_div = browser.find_element_by_xpath('//pre') assert "params_examples\\\\.track_find_test\\\\(1\\\\)" in pre_div.text # There's an unignore button that moves the element back to the main table unignore_button = [button for button in browser.find_elements_by_xpath('//button') if button.text == 'Unignore'][0] unignore_button.click() browser.find_elements_by_xpath('//table[1]/tbody/tr[2]') # wait until the table has two rows table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr') assert len(table_rows) == 2 # Check that a plot of some sort appears on mouseover. The # page needs to be scrolled first so that the mouseover popup # has enough space to appear. regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)') y = regression_1.location['y'] browser.execute_script('window.scrollTo(0, {0})'.format(y - 200)) chain = ActionChains(browser) chain.move_to_element(regression_1) chain.perform() popover = browser.find_element_by_css_selector('div.popover-content') flotplot = browser.find_element_by_css_selector('canvas.flot-base')
def test_publish(tmpdir): tmpdir = six.text_type(tmpdir) os.chdir(tmpdir) result_dir = join(tmpdir, 'sample_results') os.makedirs(result_dir) os.makedirs(join(result_dir, 'cheetah')) # Synthesize history with two branches that both have commits result_files = [fn for fn in os.listdir(join(RESULT_DIR, 'cheetah')) if fn.endswith('.json') and fn != 'machine.json'] master_values = list(range(len(result_files)*2//3)) branch_values = list(range(len(master_values), len(result_files))) dvcs = tools.generate_test_repo(tmpdir, master_values, 'git', [('master~6', 'some-branch', branch_values)]) # Copy and modify result files, fixing commit hashes and setting result # dates to distinguish the two branches master_commits = dvcs.get_branch_hashes('master') only_branch = [x for x in dvcs.get_branch_hashes('some-branch') if x not in master_commits] commits = master_commits + only_branch for k, item in enumerate(zip(result_files, commits)): fn, commit = item src = join(RESULT_DIR, 'cheetah', fn) dst = join(result_dir, 'cheetah', commit[:8] + fn[8:]) data = util.load_json(src, cleanup=False) data['commit_hash'] = commit if commit in only_branch: data['date'] = -k else: data['date'] = k util.write_json(dst, data) shutil.copyfile(join(RESULT_DIR, 'benchmarks.json'), join(result_dir, 'benchmarks.json')) shutil.copyfile(join(RESULT_DIR, 'cheetah', 'machine.json'), join(result_dir, 'cheetah', 'machine.json')) # Publish the synthesized data conf = config.Config.from_json( {'benchmark_dir': BENCHMARK_DIR, 'results_dir': result_dir, 'html_dir': join(tmpdir, 'html'), 'repo': dvcs.path, 'project': 'asv'}) Publish.run(conf) # Check output assert isfile(join(tmpdir, 'html', 'index.html')) assert isfile(join(tmpdir, 'html', 'index.json')) assert isfile(join(tmpdir, 'html', 'asv.js')) assert isfile(join(tmpdir, 'html', 'asv.css')) assert not isdir(join(tmpdir, 'html', 'graphs', 'Cython', 'arch-x86_64', 'branch-some-branch')) index = util.load_json(join(tmpdir, 'html', 'index.json')) assert index['params']['branch'] == ['master'] def check_file(branch): fn = join(tmpdir, 'html', 'graphs', 'Cython', 'arch-x86_64', 'branch-' + branch, 'cpu-Intel(R) Core(TM) i5-2520M CPU @ 2.50GHz (4 cores)', 'machine-cheetah', 'numpy-1.8', 'os-Linux (Fedora 20)', 'python-2.7', 'ram-8.2G', 'time_coordinates.time_latitude.json') data = util.load_json(fn, cleanup=False) if branch == 'master': # we set all dates positive for master above assert all(x[0] >= 0 for x in data) else: # we set some dates negative for some-branch above assert any(x[0] < 0 for x in data) and any(x[0] >= 0 for x in data) check_file("master") # Publish with branches set in the config conf.branches = ['master', 'some-branch'] Publish.run(conf) # Check output check_file("master") check_file("some-branch") index = util.load_json(join(tmpdir, 'html', 'index.json')) assert index['params']['branch'] == ['master', 'some-branch']
def create_benchmark_dataframe(group_by="name", use_branch_names=False): # if we are in an asv subprocess, use ASV_CONF_DIR to load the config repo_dirname = os.environ.get("ASV_CONF_DIR", _find_asv_root()) config_path = os.path.join(repo_dirname, "asv.conf.json") config = Config.load(config_path) # results_dir is a relative path to the benchmarks repository. If the # directory where the code is run is not the benchmark repository, then # loading the results will fail. config.results_dir = os.path.join(repo_dirname, "results") benchmarks = Benchmarks.load(config) results = defaultdict(dict) metadata_levels = [ "type", "name", "class", "file", "version", "commit_hash", "date", ] if isinstance(group_by, str): group_by = [group_by] levels_to_group_by = group_by levels_to_concat_on = [ l for l in metadata_levels if l not in levels_to_group_by ] commit_to_branch_map = _get_commit_to_branch_map(config.repo) for single_env_result in Publish.iter_results(config, benchmarks): benchmark_metadata = { "version": single_env_result._params["python"], "commit_hash": single_env_result._commit_hash, "date": single_env_result._date, } if use_branch_names: benchmark_metadata["commit_hash"] = commit_to_branch_map.get( benchmark_metadata["commit_hash"], benchmark_metadata["commit_hash"]) for b_name, params in single_env_result._benchmark_params.items(): unquoted_params = _remove_quotes(params) filename, classname, benchname = b_name.split(".") _benchmark = benchmarks[b_name] b_type, param_names = _benchmark["type"], _benchmark["param_names"] benchmark_metadata.update({ "type": b_type, "file": filename, "class": classname, "name": benchname, }) values_to_group_by = tuple( [benchmark_metadata[key] for key in levels_to_group_by]) values_to_concat_on = tuple( [benchmark_metadata[key] for key in levels_to_concat_on]) # this is dangerous because we there is no reason the results # order follow the carthesian product of the parameter space, # however empirically it seems to be the case params_with_infered_types = [] _results = single_env_result._results[b_name] for params in unquoted_params: params_with_infered_types.append( pd.to_numeric(params, errors="ignore")) if params_with_infered_types != []: mi = pd.MultiIndex.from_product(params_with_infered_types, names=param_names) else: # benchmark is not parametrized, make index a simple range # index mi = pd.RangeIndex(len(_results)) if len(_results) != len(mi): # if a benchmark fails, single_env_result._results[b_name] # only consists of [None] assert _results == [None], 'unexpected benchmark result' continue _results = pd.Series(_results, index=mi) _results.dropna(inplace=True) results[values_to_group_by][values_to_concat_on] = _results clean_result = {} for k, v in results.items(): if len(k) == 1: # if key if a list of length one, convert it to a string by taking # its only element clean_result[k[0]] = pd.concat(v, names=levels_to_concat_on) elif len(k) == 0: # if key is of length 0, there is only one element, so, return the # underlying dict clean_result = pd.concat(v, names=levels_to_concat_on) else: clean_result[k] = pd.concat(v, names=levels_to_concat_on) return clean_result