def _test_run_branches(tmpdir, dvcs, conf, machine_file, range_spec, branches, initial_commit): # Find the current head commits for each branch commits = [initial_commit] for branch in branches: commits.append(dvcs.get_hash(branch)) # Run tests Run.run(conf, range_spec=range_spec, _machine_file=machine_file, quick=True) # Check that files for all commits expected were generated expected = set(['machine.json']) for commit in commits: for psver in ['0.3.1', '0.3.3']: expected.add('{0}-py{1[0]}.{1[1]}-colorama{2}-six.json'.format( commit[:8], sys.version_info, psver)) result_files = os.listdir(join(tmpdir, 'results_workflow', 'orangutan')) if range_spec == 'NEW': assert set(result_files) == expected elif range_spec == 'ALL': assert set(expected).difference(result_files) == set([]) else: raise ValueError()
def init_results(): results_dir = os.path.join(tmpdir, 'results_workflow') if os.path.isdir(results_dir): shutil.rmtree(results_dir) Run.run(conf, range_spec=initial_commit+"^!", bench=["time_secondary.track_value"], _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, skip_successful=True, skip_failed=True)
def test_run_python_same(capsys, basic_conf): tmpdir, local, conf = basic_conf # Test Run runs with python=same Run.run(conf, _machine_file=join(tmpdir, 'asv-machine.json'), python="same") text, err = capsys.readouterr() assert re.search("time_exception.*failed", text, re.S) assert re.search(r"Running time_secondary.track_value\s+42.0", text) # Check that it did not clone or install assert "Cloning" not in text assert "Installing" not in text
def test_run_publish(capfd, basic_conf): tmpdir, local, conf, machine_file = basic_conf # Tests a typical complete run/publish workflow Run.run(conf, range_spec="master~5..master", steps=2, _machine_file=machine_file, quick=True, show_stderr=True) text, err = capfd.readouterr() assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5 assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2 assert 'asv: benchmark timed out (timeout 0.1s)' in text Publish.run(conf) assert isfile(join(tmpdir, 'html', 'index.html')) assert isfile(join(tmpdir, 'html', 'index.json')) assert isfile(join(tmpdir, 'html', 'asv.js')) assert isfile(join(tmpdir, 'html', 'asv.css')) # Check parameterized test json data format filename = glob.glob(join(tmpdir, 'html', 'graphs', 'arch-x86_64', 'branch-master', 'colorama-0.3.3', 'cpu-Blazingly fast', 'machine-orangutan', 'os-GNU', 'Linux', 'python-*', 'ram-128GB', 'six', 'params_examples.time_skip.json'))[0] with open(filename, 'r') as fp: data = json.load(fp) assert len(data) == 2 assert isinstance(data[0][0], six.integer_types) # date assert len(data[0][1]) == 3 assert len(data[1][1]) == 3 assert isinstance(data[0][1][0], float) assert isinstance(data[0][1][1], float) assert data[0][1][2] is None # Check that the skip options work capfd.readouterr() Run.run(conf, range_spec="master~5..master", steps=2, _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, skip_successful=True, skip_failed=True) Run.run(conf, range_spec="master~5..master", steps=2, _machine_file=join(tmpdir, 'asv-machine.json'), quick=True, skip_existing_commits=True) text, err = capfd.readouterr() assert 'Running benchmarks.' not in text # Check EXISTING works Run.run(conf, range_spec="EXISTING", _machine_file=machine_file, quick=True) # Remove the benchmarks.json file to make sure publish can # regenerate it os.remove(join(tmpdir, "results_workflow", "benchmarks.json")) Publish.run(conf)
def test_workflow(tmpdir): # Tests a typical complete run/publish workflow tmpdir = six.text_type(tmpdir) local = abspath(dirname(__file__)) os.chdir(tmpdir) shutil.copyfile(join(local, 'asv-machine.json'), join(tmpdir, 'asv-machine.json')) conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': 'https://github.com/spacetelescope/asv.git', 'project': 'asv', 'matrix': { "six": [None], "psutil": ["1.2", "1.1"] } }) Run.run(conf, range_spec="initial..master", steps=2, _machine_file=join(tmpdir, 'asv-machine.json'), quick=True) assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5 assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2 Publish.run(conf) assert exists(join(tmpdir, 'html', 'index.html')) assert exists(join(tmpdir, 'html', 'index.json')) assert exists(join(tmpdir, 'html', 'asv.js')) assert exists(join(tmpdir, 'html', 'asv.css')) Run.run(conf, range_spec="EXISTING", _machine_file=join(tmpdir, 'asv-machine.json'), quick=True) # Remove the benchmarks.json file to make sure publish can # regenerate it os.remove(join(tmpdir, "results_workflow", "benchmarks.json")) Publish.run(conf)
def test_workflow(tmpdir): # Tests a typical complete run/publish workflow tmpdir = six.text_type(tmpdir) local = abspath(dirname(__file__)) os.chdir(tmpdir) conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': 'https://github.com/spacetelescope/asv.git', 'project': 'asv', 'matrix': { "six": [None], "psutil": ["1.2", "1.1"] } }) Run.run(conf, range_spec="initial..master", steps=2, _machine_file=join(local, 'asv-machine.json'), quick=True) assert len(os.listdir(join(tmpdir, 'results_workflow', 'orangutan'))) == 5 assert len(os.listdir(join(tmpdir, 'results_workflow'))) == 2 Publish.run(conf) assert exists(join(tmpdir, 'html', 'index.html')) assert exists(join(tmpdir, 'html', 'index.json')) assert exists(join(tmpdir, 'html', 'asv.js')) assert exists(join(tmpdir, 'html', 'asv.css')) Run.run(conf, range_spec="EXISTING", _machine_file=join(local, 'asv-machine.json'), quick=True) # Remove the benchmarks.json file to make sure publish can # regenerate it os.remove(join(tmpdir, "results_workflow", "benchmarks.json")) Publish.run(conf)
def test_run_python_same(basic_conf): tmpdir, local, conf = basic_conf # Test Run runs with python=same s = StringIO() stdout = sys.stdout try: sys.stdout = s Run.run(conf, _machine_file=join(tmpdir, 'asv-machine.json'), python="same") finally: sys.stdout = stdout s.seek(0) text = s.read() assert re.search("time_exception.*failed", text, re.S) assert re.search(r"Running time_secondary.track_value\s+42.0", text) # Check that it did not clone or install assert "Cloning" not in text assert "Installing" not in text
def basic_html(request): tmpdir = tempfile.mkdtemp() request.addfinalizer(lambda: shutil.rmtree(tmpdir)) local = abspath(dirname(__file__)) cwd = os.getcwd() os.chdir(tmpdir) try: machine_file = join(tmpdir, 'asv-machine.json') shutil.copyfile(join(local, 'asv-machine.json'), machine_file) dvcs = tools.generate_test_repo(tmpdir, list(range(10))) repo_path = dvcs.path conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': repo_path, 'dvcs': 'git', 'project': 'asv', 'matrix': { "six": [None], "colorama": ["0.3.1", "0.3.3"] } }) Run.run(conf, range_spec="master~5..master", steps=3, _machine_file=machine_file, quick=True) Publish.run(conf) finally: os.chdir(cwd) return conf, dvcs
def test_format_durations(): durations = {'foo': 1, 'bar': 2, 'quux': 3} msg = Run.format_durations(durations, 2) expected = textwrap.dedent("""\ =========== ================ benchmark total duration ----------- ---------------- quux 3.00s bar 2.00s ... ... total 6.00s =========== ================""") assert msg == expected
def test_web_regressions(browser, tmpdir): from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver import ActionChains tmpdir = six.text_type(tmpdir) local = abspath(dirname(__file__)) cwd = os.getcwd() os.chdir(tmpdir) try: machine_file = join(tmpdir, 'asv-machine.json') shutil.copyfile(join(local, 'asv-machine.json'), machine_file) values = [[x]*2 for x in [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2]] dvcs = tools.generate_test_repo(tmpdir, values) repo_path = dvcs.path first_tested_commit_hash = dvcs.get_hash('master~14') conf = config.Config.from_json({ 'env_dir': join(tmpdir, 'env'), 'benchmark_dir': join(local, 'benchmark'), 'results_dir': join(tmpdir, 'results_workflow'), 'html_dir': join(tmpdir, 'html'), 'repo': repo_path, 'dvcs': 'git', 'project': 'asv', 'matrix': {}, 'regressions_first_commits': { '.*': first_tested_commit_hash }, }) Run.run(conf, range_spec="ALL", bench='params_examples.track_find_test', _machine_file=machine_file, show_stderr=True, quick=True) Publish.run(conf) finally: os.chdir(cwd) bad_commit_hash = dvcs.get_hash('master~9') with tools.preview(conf.html_dir) as base_url: browser.get(base_url) regressions_btn = browser.find_element_by_link_text('Show regressions') regressions_btn.click() # Check that the expected links appear in the table regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)') regression_2 = browser.find_element_by_link_text('params_examples.track_find_test(2)') bad_hash_link = browser.find_element_by_link_text(bad_commit_hash[:8]) href = regression_1.get_attribute('href') assert '/#params_examples.track_find_test?' in href assert 'time=' in href # Sort the tables vs. benchmark name (PhantomJS doesn't allow doing it via actionchains) browser.execute_script("$('thead th').eq(0).stupidsort('asc')") WebDriverWait(browser, 5).until(EC.text_to_be_present_in_element( ('xpath', '//table[1]/tbody/tr[1]/td[1]'), 'params_examples.track_find_test(1)' )) # Check the contents of the table table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr') assert len(table_rows) == 2 cols1 = [td.text for td in table_rows[0].find_elements_by_xpath('td')] cols2 = [td.text for td in table_rows[1].find_elements_by_xpath('td')] assert cols1[0] == 'params_examples.track_find_test(1)' assert cols2[0] == 'params_examples.track_find_test(2)' assert re.match(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d+Z$', cols1[1]) assert re.match(r'^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d.\d+Z$', cols2[1]) assert cols1[2:] == [bad_commit_hash[:8], '2.00x', '1.00', '2.00', 'Ignore'] assert cols2[2:] == [bad_commit_hash[:8], '2.00x', '1.00', '2.00', 'Ignore'] # Check that the ignore buttons work as expected buttons = [button for button in browser.find_elements_by_xpath('//button') if button.text == 'Ignore'] buttons[0].click() # The button should disappear, together with the link WebDriverWait(browser, 5).until_not(EC.visibility_of(buttons[0])) WebDriverWait(browser, 5).until_not(EC.visibility_of(regression_1)) table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr') assert len(table_rows) == 1 # There's a second button for showing the links, clicking # which makes the elements reappear show_button = [button for button in browser.find_elements_by_xpath('//button') if button.text == 'Show ignored regressions...'][0] show_button.click() regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)') WebDriverWait(browser, 5).until(EC.visibility_of(regression_1)) table_rows = browser.find_elements_by_xpath('//table[2]/tbody/tr') assert len(table_rows) == 1 # There's a config sample element pre_div = browser.find_element_by_xpath('//pre') assert "params_examples\\\\.track_find_test\\\\(1\\\\)" in pre_div.text # There's an unignore button that moves the element back to the main table unignore_button = [button for button in browser.find_elements_by_xpath('//button') if button.text == 'Unignore'][0] unignore_button.click() browser.find_elements_by_xpath('//table[1]/tbody/tr[2]') # wait until the table has two rows table_rows = browser.find_elements_by_xpath('//table[1]/tbody/tr') assert len(table_rows) == 2 # Check that a plot of some sort appears on mouseover. The # page needs to be scrolled first so that the mouseover popup # has enough space to appear. regression_1 = browser.find_element_by_link_text('params_examples.track_find_test(1)') y = regression_1.location['y'] browser.execute_script('window.scrollTo(0, {0})'.format(y - 200)) chain = ActionChains(browser) chain.move_to_element(regression_1) chain.perform() popover = browser.find_element_by_css_selector('div.popover-content') flotplot = browser.find_element_by_css_selector('canvas.flot-base')