def test_vara_cs_cleanup_error(self):
"""Test vara-cs cleanup error."""
runner = CliRunner()
vara_cfg()["paper_config"]["current_config"] = "test_cleanup_error"
save_config()
load_paper_config()
importlib.reload(driver_casestudy)
result = runner.invoke(
driver_casestudy.main, ['cleanup', 'all', '--error']
)
self.assertEqual(0, result.exit_code, result.stdout)
self.assertFalse(
Path(
vara_cfg()["result_dir"].value +
"/brotli/CRE-CR-brotli-all-6c47009892_5d26c7ff-6d27-478f-bcd1"
"-99e8e8e97f16_cerror.txt"
).exists()
)
self.assertFalse(
Path(
vara_cfg()["result_dir"].value +
"/brotli/CRE-CR-brotli-all-aaa4424d9b_5d26c7ff-6d27-478f-bcd1-"
"99e8e8e97f16_failed.txt"
).exists()
)
self.assertTrue(
Path(
vara_cfg()["result_dir"].value +
"/brotli/CRE-CR-brotli-brotli-21ac39f7c8_34d4d1b5-7212-4244-"
"9adc-b19bff599cf1_success.yaml"
).exists()
)
def _pc_create(paper_config: str) -> None:
"""paper_config:
Path to the new paper config.
Relative paths are interpreted relative to the current
`paper_config/folder`."""
pc_path: Path = Path(paper_config)
if not pc_path.is_absolute():
current_folder = vara_cfg()["paper_config"]["folder"].value
if current_folder is None:
pc_path = Path(vara_cfg()["config_file"].value
).parent / "paper_configs" / pc_path
else:
pc_path = Path(current_folder) / pc_path
if pc_path.exists():
LOG.error(f"Cannot create paper config at: {pc_path} "
"(Path already exists).")
return
folder = pc_path.parent
current_config = pc_path.name
LOG.info(
f"Creating new paper config {current_config} at location {folder}.")
pc_path.mkdir(parents=True)
vara_cfg()["paper_config"]["folder"] = str(folder)
vara_cfg()["paper_config"]["current_config"] = str(current_config)
save_config()
def test_vara_cs_gen_to_extend_new_stage(self):
"""Test the extend-functionality of vara-cs gen."""
runner = CliRunner()
vara_cfg()["paper_config"]["current_config"] = "test_ext"
save_config()
load_paper_config()
old_commit = 'ef364d3abc5647111c5424ea0d83a567e184a23b'
new_commit = '6c6da57ae2aa962aabde6892442227063d87e88c'
result = runner.invoke(
driver_casestudy.main, [
'gen', '-p', 'xz', '--new-stage', '--merge-stage', 'test',
'select_specific', new_commit
]
)
self.assertEqual(0, result.exit_code, result.exception)
case_study_path = Path(
vara_cfg()["paper_config"]["folder"].value +
"/test_ext/xz_0.case_study"
)
self.assertTrue(case_study_path.exists())
case_study = load_case_study_from_file(case_study_path)
self.assertTrue(
case_study.revisions.__contains__(FullCommitHash(old_commit))
)
self.assertTrue(
case_study.revisions.__contains__(FullCommitHash(new_commit))
)
self.assertEqual(2, case_study.num_stages)
self.assertEqual('test', case_study.stages[1].name)
def __init__(self) -> None:
if hasattr(Qt, 'AA_EnableHighDpiScaling'):
QApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True)
if hasattr(Qt, 'AA_UseHighDpiPixmaps'):
QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True)
self.app = QApplication(sys.argv)
if settings.vara_cfg()["config_file"].value is None:
err = QMessageBox()
err.setIcon(QMessageBox.Warning)
err.setWindowTitle("Missing config file.")
err.setText(
"Could not find VaRA config file.\n"
"Should we create a config file in the current folder?"
)
err.setStandardButtons(
tp.cast(
QMessageBox.StandardButtons,
QMessageBox.Yes | QMessageBox.No
)
)
answer = err.exec_()
if answer == QMessageBox.Yes:
settings.save_config()
else:
sys.exit()
self.main_window = MainWindow()
def test_store_artefact(self):
"""Test whether `vara-plot` can store artefacts."""
# setup config
vara_cfg()['paper_config']['current_config'] = "test_artefacts_driver"
load_paper_config()
save_config()
artefacts_file_path = get_paper_config().path / "artefacts.yaml"
artefacts_file_path.unlink()
# vara-plot
runner = CliRunner()
result = runner.invoke(driver_plot.main, [
"--save-artefact=PC Overview", "--plot-dir=foo",
"pc-overview-plot", "--report-type=EmptyReport"
])
self.assertEqual(0, result.exit_code, result.exception)
self.assertTrue(artefacts_file_path.exists())
# load new artefact file
load_paper_config()
artefacts = list(get_paper_config().artefacts)
self.assertEqual(1, len(artefacts))
artefact = artefacts[0]
self.assertIsInstance(artefact, PlotArtefact)
self.assertEqual("PC Overview", artefact.name)
def setup(self, source_folder: tp.Optional[Path], install_prefix: Path,
version: tp.Optional[int]) -> None:
"""
Setup the research tool VaRA with it's code base. This method sets up
all relevant config variables, downloads repositories via the
``CodeBase``, checkouts the correct branches and prepares the research
tool to be built.
Args:
source_folder: location to store the code base in
install_prefix: Installation prefix path
version: Version to setup
"""
cfg = vara_cfg()
if source_folder:
cfg["vara"]["llvm_source_dir"] = str(source_folder)
cfg["vara"]["llvm_install_dir"] = str(install_prefix)
if version:
version = int(version)
cfg["vara"]["version"] = version
else:
version = int(cfg["vara"]["version"].value)
save_config()
print(f"Setting up VaRA in {self.source_location()}")
use_dev_branches = cfg["vara"]["developer_version"].value
self.code_base.clone(self.source_location())
self.code_base.setup_vara_remotes()
self.code_base.checkout_vara_version(version, use_dev_branches)
self.code_base.setup_submodules()
self.code_base.setup_build_link()
def setup(self, source_folder: tp.Optional[Path], install_prefix: Path,
version: tp.Optional[int]) -> None:
"""
Setup the research tool phasar with it's code base. This method sets up
all relevant config variables, downloads repositories via the
``CodeBase``, checkouts the correct branches and prepares the research
tool to be built.
Args:
source_folder: location to store the code base in
install_prefix: Installation prefix path
version: Version to setup
"""
cfg = vara_cfg()
if source_folder:
cfg["phasar"]["source_dir"] = str(source_folder)
cfg["phasar"]["install_dir"] = str(install_prefix)
save_config()
print(f"Setting up phasar in {self.source_location()}")
use_dev_branch = cfg["phasar"]["developer_version"].value
self.code_base.clone(self.source_location())
self.code_base.checkout_phasar_version(use_dev_branch)
self.code_base.setup_submodules()
def test_bb_run_slurm_and_container(self, mock_sbatch,
mock_build_images) -> None:
runner = CliRunner()
vara_cfg()['paper_config']['current_config'] = "test_revision_lookup"
# needed so we see the paper config
load_paper_config()
# needed so benchbuild sees the paper config
save_config()
bb_cfg()["slurm"]["template"] = str(
Path(str(vara_cfg()["benchbuild_root"])) /
"slurm_container.sh.inc")
save_bb_config()
# Easiest way to configure slurm + container is 'vara-container'
# As a side-effect, this command is now even more tested :)
prepare_result = runner.invoke(driver_container.main,
["prepare-slurm"])
self.assertEqual(0, prepare_result.exit_code, prepare_result.exception)
self.assertTrue(Path(str(bb_cfg()["slurm"]["template"])).exists())
result = runner.invoke(driver_run.main,
["--slurm", "--container", "-E", "JustCompile"])
self.assertEqual(0, result.exit_code, result.exception)
self.assertTrue((Path(str(vara_cfg()["benchbuild_root"])) /
"JustCompile-slurm.sh").exists())
def test_vara_config_set(self):
runner = CliRunner()
vara_cfg()["artefacts"]["artefacts_dir"] = "test"
save_config()
runner.invoke(driver_config.main,
["set", "artefacts/artefacts_dir=artefacts"])
self.assertEqual("artefacts",
vara_cfg()["artefacts"]["artefacts_dir"].value)
def _pc_set(paper_config: tp.Optional[Path]) -> None:
if not paper_config:
pc_folder_path = Path(
get_value_or_default(vara_cfg()["paper_config"], "folder",
str(get_varats_base_folder())))
if not (pc_folder_path.exists() and pc_folder_path.is_dir()):
LOG.error(f"Paper config folder not set: {pc_folder_path} "
"(Path does not exist or is no directory).")
return
paper_configs = _get_paper_configs(pc_folder_path)
if not paper_configs:
LOG.error(f"Could not find paper configs in: {pc_folder_path}")
return
raw_pc_path = None
def set_pc_path(choice: str) -> None:
nonlocal raw_pc_path
raw_pc_path = choice
current_config = None
try:
current_config = get_paper_config().path.name
except ConfigurationLookupError:
# No paper config specified in the varats config file
pass
try:
cli_list_choice(
"Choose a number to select a paper config", paper_configs,
lambda x: f"{x} *"
if current_config and x == current_config else x, set_pc_path)
except EOFError:
return
if raw_pc_path is None:
raise AssertionError("Choice should always return a value")
paper_config = Path(raw_pc_path)
paper_config = Path(paper_config)
if not paper_config.is_absolute():
paper_config = Path(
vara_cfg()["paper_config"]["folder"].value) / paper_config
if not (paper_config.exists() and paper_config.is_dir()):
LOG.error(f"Not a paper config: {paper_config} "
"(Path does not exist or is no directory).")
return
folder = paper_config.parent
current_config = paper_config.name
LOG.info(
f"Current paper config is now {current_config} at location {folder}.")
vara_cfg()["paper_config"]["folder"] = str(folder)
vara_cfg()["paper_config"]["current_config"] = str(current_config)
save_config()
def main() -> None:
"""
Main function for working with paper configs.
`vara-pc`
"""
initialize_cli_tool()
if vara_cfg()["paper_config"]["folder"].value is None:
# Setup default paper config path when none exists
vara_cfg()["paper_config"]["folder"] = str(
Path('paper_configs').absolute())
save_config()
def test_bb_run_all(self) -> None:
runner = CliRunner()
vara_cfg()['paper_config']['current_config'] = "test_artefacts_driver"
# needed so we see the paper config
load_paper_config()
# needed so benchbuild sees the paper config
save_config()
result = runner.invoke(driver_run.main, ["-p", "-E", "JustCompile"])
self.assertEqual(0, result.exit_code, result.exception)
match = self.__NUM_ACTIONS_PATTERN.search(result.stdout)
if not match:
self.fail("Could not parse benchbuild output")
self.assertEqual("51", match.group(1))
def test_cli_option_converter(self):
"""Test whether CLI option conversion works correctly."""
# setup config
vara_cfg()['paper_config']['current_config'] = "test_artefacts_driver"
load_paper_config()
save_config()
plot_generator = CaseStudyOverviewGenerator(
PlotConfig.from_kwargs(view=False),
report_type=EmptyReport,
case_study=get_loaded_paper_config().get_case_studies("xz")[0])
artefact = PlotArtefact.from_generator("CS Overview", plot_generator,
CommonPlotOptions.from_kwargs())
artefact_dict = artefact.get_dict()
self.assertEqual("xz_0", artefact_dict["case_study"])
self.assertEqual("EmptyReport", artefact_dict["report_type"])
def upgrade(self) -> None:
"""Upgrade the research tool to a newer version."""
new_version = self.find_highest_sub_prj_version("vara-llvm-project")
if new_version != (math.ceil(
self.find_highest_sub_prj_version("VaRA") / 10) * 10):
raise AssertionError(
"vara-llvm-project and vara tool out of sync.")
if str(vara_cfg()["vara"]["version"]) != str(new_version):
self.code_base.checkout_vara_version(new_version, True)
vara_cfg()["vara"]["version"] = new_version
save_config()
self.code_base.pull()
def test_vara_cs_cleanup_regex(self):
"""Test vara-cs cleanup regex."""
runner = CliRunner()
vara_cfg()["paper_config"]["current_config"] = "test_cleanup_regex"
save_config()
load_paper_config()
importlib.reload(driver_casestudy)
result = runner.invoke(
driver_casestudy.main, ['cleanup', 'regex', '-f', '.*'], 'y'
)
self.assertEqual(0, result.exit_code, result.exception)
self.assertFalse(
Path(
vara_cfg()["result_dir"].value +
"/brotli/CRE-CR-brotli-all-6c47009892_5d26c7ff-6d27-478f-bcd1-"
"99e8e8e97f16_cerror.txt"
).exists()
)
self.assertFalse(
Path(
vara_cfg()["result_dir"].value +
"/brotli/CRE-CR-brotli-all-aaa4424d9b_5d26c7ff-6d27-478f-bcd1-"
"99e8e8e97f16_failed.txt"
).exists()
)
self.assertFalse(
Path(
vara_cfg()["result_dir"].value +
"/brotli/CRE-CR-brotli-brotli-21ac39f7c8_34d4d1b5-7212-4244-"
"9adc-b19bff599cf1_success.yaml"
).exists()
)
self.assertFalse(
Path(
vara_cfg()["result_dir"].value +
"/brotli/CRE-CR-brotli-brotli-21ac39f7c8_34d4d1b5-7212-4244-"
"9adc-b19bff599142_success.yaml"
).exists()
)
self.assertTrue(
Path(
vara_cfg()["result_dir"].value +
"/gravity/BVRE_NoOptTBAA-BVR_NoOpt_TBAA-gravity-gravity-"
"b51227de55_8bc2ac4c-b6e3-43d1-aff9-c6b32126b155_success.txt"
).exists()
)
def test_plot(self):
"""Test whether `vara-plot` generates a plot."""
# setup config
vara_cfg()['paper_config']['current_config'] = "test_artefacts_driver"
load_paper_config()
save_config()
plot_base_dir = Path(str(vara_cfg()['plots']['plot_dir']))
# vara-plot
runner = CliRunner()
result = runner.invoke(driver_plot.main, [
"--plot-dir=foo", "pc-overview-plot", "--report-type=EmptyReport"
])
self.assertEqual(0, result.exit_code, result.exception)
self.assertTrue((plot_base_dir / "foo" /
"paper_config_overview_plot.svg").exists())
def setup(self, source_folder: tp.Optional[Path], install_prefix: Path,
version: tp.Optional[int]) -> None:
"""
Setup the research tool SZZUnleashed with it's code base.
Args:
source_folder: location to store the code base in
install_prefix: Installation prefix path
version: Version to setup
"""
cfg = vara_cfg()
if source_folder:
cfg["szzunleashed"]["source_dir"] = str(source_folder)
cfg["szzunleashed"]["install_dir"] = str(install_prefix)
save_config()
print(f"Setting up SZZUnleashed in {self.source_location()}")
self.code_base.clone(self.source_location())
def __enter__(self) -> Path:
os.chdir(self.__tmp_dir.name)
vara_cfg = settings.create_new_varats_config()
bb_settings.setup_config(vara_cfg)
# pylint: disable=protected-access
settings._CFG = vara_cfg
settings.save_config()
bb_cfg = create_new_bb_config(settings.vara_cfg(), True)
# make new bb_cfg point to old tmp to avoid multiple git clones
bb_cfg["tmp_dir"] = str(self.__old_bb_config["tmp_dir"])
settings.save_bb_config(bb_cfg)
# pylint: disable=protected-access
settings._BB_CFG = bb_cfg
for test_input in self.__test_inputs:
test_input.copy_to_env(self.__tmp_path)
return self.__tmp_path
def test_vara_cs_status(self):
"""Test for vara-cs status."""
runner = CliRunner()
vara_cfg()["paper_config"]["current_config"] = "test_status"
save_config()
load_paper_config()
result = runner.invoke(driver_casestudy.main, ['status', 'JustCompile'])
self.assertEqual(0, result.exit_code, result.exception)
self.assertEqual(
"CS: xz_0: ( 0/5) processed [0/0/0/0/3/2]\n"
" c5c7ceb08a [Missing]\n"
" ef364d3abc [Missing]\n"
" 2f0bc9cd40 [Missing]\n"
" 7521bbdc83 [Blocked]\n"
" 10437b5b56 [Blocked]\n\n"
"---------------------------------------------"
"-----------------------------------\n"
"Total: ( 0/5) processed [0/0/0/0/3/2]\n", result.stdout
)
def main(bb_root: tp.Optional[str] = None,
test_projects: bool = False) -> None:
"""
Main function for the benchbuild config creator.
`vara-gen-bbconfig`
"""
if vara_cfg()["config_file"].value is None:
if cli_yn_choice("Error! No VaRA config found. Should we create one?"):
save_config()
else:
sys.exit()
if bb_root is not None:
if os.path.isabs(str(bb_root)):
bb_root_path = str(bb_root)
else:
bb_root_path = os.path.dirname(str(vara_cfg()["config_file"])) + \
"/" + str(bb_root)
LOG.info(f"Setting BB path to: {bb_root_path}")
vara_cfg()["benchbuild_root"] = bb_root_path
save_config()
if vara_cfg()["benchbuild_root"].value is None:
vara_cfg()["benchbuild_root"] = os.path.dirname(str(
vara_cfg()["config_file"])) \
+ "/benchbuild"
LOG.info(f"Setting BB path to: {vara_cfg()['benchbuild_root']}")
save_config()
bb_cfg = create_new_bb_config(vara_cfg(), test_projects)
save_bb_config(bb_cfg)
def __config_set(config_values: tp.List[str]) -> None:
"""
KEY=VALUE Key-Value pairs of configuration options and values.
Specify the config options like paths, e.g., paper_config/folder. Do not put
spaces before or after the '=' sign; if a value contains spaces, you should
define it with double quotes: foo="bar baz".
"""
if config_values:
rewritten_config_values = {
e[0].replace('-', '_'): e[1]
for e in [arg.split("=") for arg in config_values]
}
else:
rewritten_config_values = {}
for option, value in rewritten_config_values.items():
option_path = option.split("/")
config = __get_config_for_path(option_path[:-1])
config[option_path[-1]] = value
save_config()
def __init__(self, base_dir: Path) -> None:
super().__init__("phasar", [BuildType.DEV], PhasarCodeBase(base_dir))
vara_cfg()["phasar"]["source_dir"] = str(base_dir)
save_config()
def __init__(self, base_dir: Path) -> None:
super().__init__("SZZUnleashed", [BuildType.DEV],
SZZUnleashedCodeBase(base_dir))
vara_cfg()["szzunleashed"]["source_dir"] = str(base_dir)
save_config()
def __init__(self, base_dir: Path) -> None:
super().__init__("VaRA", [BuildType.DEV], VaRACodeBase(base_dir))
vara_cfg()["vara"]["llvm_source_dir"] = str(base_dir)
save_config()
def config() -> None:
"""Only create a VaRA-TS config file."""
save_config()
def _save_config() -> None:
"""Save current config to file."""
save_config()
